 {
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# First ensemble with NSL-KDD\n",
    "# Parameters\n",
    "\n",
    "#----------------------------------------------\n",
    "# 0 for not using it as base learner\n",
    "# 1 for using it as base learner\n",
    "\n",
    "use_model_ada = 1 \n",
    "use_model_dnn = 1 \n",
    "use_model_mlp = 1 \n",
    "use_model_lgbm = 1 \n",
    "use_model_rf = 1 \n",
    "use_model_svm = 1\n",
    "use_model_knn = 1 \n",
    "#----------------------------------------------\n",
    "# 0 for training the model\n",
    "# 1 for using the saved version of the model\n",
    "\n",
    "load_model_ada = 0 \n",
    "load_model_dnn = 0 \n",
    "load_model_mlp = 0 \n",
    "load_model_lgbm = 0 \n",
    "load_model_rf = 0 \n",
    "load_model_svm = 0\n",
    "load_model_knn = 0 \n",
    "#----------------------------------------------\n",
    "\n",
    "# load_model_ada = 1\n",
    "# load_model_dnn = 1 \n",
    "# load_model_mlp = 1 \n",
    "# load_model_lgbm = 1 \n",
    "# load_model_rf = 1 \n",
    "# load_model_svm = 1\n",
    "# load_model_knn = 1 \n",
    "#----------------------------------------------\n",
    "\n",
    "#----------------------------------------------\n",
    "# feature_selection_bit = 0 # OFF\n",
    "feature_selection_bit = 1 # On\n",
    "pick_prob = 0 # set equal one to choose the dataset with probabilities, set to 0 to choose one with the classes.\n",
    "# pick_prob = 0\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Specify the name of the output text file\n",
    "if feature_selection_bit == 0:\n",
    "\n",
    "    if pick_prob == 0:\n",
    "        output_file_name = \"ensemble_level_01_all_features_classes.txt\"\n",
    "        with open(output_file_name, \"w\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "        with open(output_file_name, \"a\") as f: print('----ensemble_level_01_all_features_classes--', file = f)\n",
    "\n",
    "    elif pick_prob == 1:\n",
    "        output_file_name = \"ensemble_level_01_all_features_probabilites.txt\"\n",
    "        with open(output_file_name, \"w\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "        with open(output_file_name, \"a\") as f: print('----ensemble_level_01_all_features_probabilites--', file = f)\n",
    "\n",
    "elif feature_selection_bit == 1:\n",
    "    if pick_prob == 0:\n",
    "        output_file_name = \"ensemble_level_01_feature_selection_classes.txt\"\n",
    "        with open(output_file_name, \"w\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "        with open(output_file_name, \"a\") as f: print('----ensemble_level_01_feature_selection_classes--', file = f)\n",
    "    elif pick_prob == 1:\n",
    "        output_file_name = \"ensemble_level_01_feature_selection_probabilites.txt\"\n",
    "        with open(output_file_name, \"w\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "        with open(output_file_name, \"a\") as f: print('----ensemble_level_01_feature_selection_probabilites--', file = f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!/usr/bin/env python\n",
    "# coding: utf-8\n",
    "\n",
    "# In[1]:\n",
    "# importing required libraries\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle # saving and loading trained model\n",
    "from os import path\n",
    "\n",
    "\n",
    "# importing required libraries for normalizing data\n",
    "from sklearn import preprocessing\n",
    "from sklearn.preprocessing import (StandardScaler, OrdinalEncoder,LabelEncoder, MinMaxScaler, OneHotEncoder)\n",
    "from sklearn.preprocessing import Normalizer, MaxAbsScaler , RobustScaler, PowerTransformer\n",
    "\n",
    "# importing library for plotting\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "from sklearn import metrics\n",
    "from sklearn.metrics import accuracy_score # for calculating accuracy of model\n",
    "from sklearn.model_selection import train_test_split # for splitting the dataset for training and testing\n",
    "from sklearn.metrics import classification_report # for generating a classification report of model\n",
    "\n",
    "from sklearn.metrics import precision_score\n",
    "from sklearn.metrics import recall_score\n",
    "from sklearn.metrics import f1_score\n",
    "\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.metrics import roc_curve, auc\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "\n",
    "from keras.layers import Dense # importing dense layer\n",
    "\n",
    "from keras.layers import Input\n",
    "from keras.models import Model\n",
    "# representation of model layers\n",
    "#from keras.utils import plot_model\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import shap\n",
    "import time\n",
    "\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import precision_score\n",
    "from sklearn.metrics import recall_score\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.metrics import balanced_accuracy_score\n",
    "from sklearn.metrics import matthews_corrcoef\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "import joblib\n",
    "from sklearn.model_selection import train_test_split\n",
    "import sklearn\n",
    "from tabulate import tabulate\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# column_features = ['dnn',\n",
    "#                    'rf',\n",
    "#                    'lgbm',\n",
    "#                 #    'ada',\n",
    "#                 #    'knn',\n",
    "#                 #    'mlp',\n",
    "#                 #    'svm',\n",
    "#                 #    'cat',\n",
    "#                    'xgb',\n",
    "#                    'label']\n",
    "# in order of importance\n",
    "# DNN\n",
    "# RF\n",
    "# XGB\n",
    "# LGBM\n",
    "\n",
    "column_features = ['dnn',\n",
    "                   'rf',\n",
    "                   'lgbm',\n",
    "                   'xgb',\n",
    "               #    'ada',\n",
    "                #    'knn',\n",
    "                #    'mlp',\n",
    "                #    'svm',\n",
    "                #    'cat',\n",
    "                   'label']\n",
    "# Assuming df is your DataFrame\n",
    "# if feature_selection_bit == 1:\n",
    "#     df_level_00_1=pd.read_csv('base_models_prob_feature_selection.csv',names=column_features)\n",
    "#     df_level_00_0=pd.read_csv('base_models_class_feature_selection.csv',names=column_features)\n",
    "\n",
    "# if feature_selection_bit == 0:\n",
    "\n",
    "#     df_level_00_1=pd.read_csv('base_models_prob_feature_selection.csv')\n",
    "#     df_level_00_0=pd.read_csv('base_models_class_feature_selection.csv')\n",
    "\n",
    "df_level_00_1=pd.read_csv('base_models_prob_feature_selection.csv')\n",
    "df_level_00_0=pd.read_csv('base_models_class_feature_selection.csv')\n",
    "\n",
    "if feature_selection_bit == 1:\n",
    "    df_level_00_0 = df_level_00_0[column_features]\n",
    "    df_level_00_1 = df_level_00_1[column_features]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dnn</th>\n",
       "      <th>rf</th>\n",
       "      <th>lgbm</th>\n",
       "      <th>xgb</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.227917</td>\n",
       "      <td>0.991146</td>\n",
       "      <td>0.999975</td>\n",
       "      <td>0.996300</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.240160</td>\n",
       "      <td>0.991146</td>\n",
       "      <td>0.999979</td>\n",
       "      <td>0.996300</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.250123</td>\n",
       "      <td>0.991146</td>\n",
       "      <td>0.999971</td>\n",
       "      <td>0.996300</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.116905</td>\n",
       "      <td>0.563712</td>\n",
       "      <td>0.994721</td>\n",
       "      <td>0.951862</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.108086</td>\n",
       "      <td>0.828218</td>\n",
       "      <td>0.999755</td>\n",
       "      <td>0.977432</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74254</th>\n",
       "      <td>0.239538</td>\n",
       "      <td>0.991146</td>\n",
       "      <td>0.999975</td>\n",
       "      <td>0.996300</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74255</th>\n",
       "      <td>0.161583</td>\n",
       "      <td>0.958180</td>\n",
       "      <td>0.999924</td>\n",
       "      <td>0.989533</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74256</th>\n",
       "      <td>0.264320</td>\n",
       "      <td>0.974195</td>\n",
       "      <td>0.999806</td>\n",
       "      <td>0.991595</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74257</th>\n",
       "      <td>0.346290</td>\n",
       "      <td>0.466003</td>\n",
       "      <td>0.897302</td>\n",
       "      <td>0.734670</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74258</th>\n",
       "      <td>0.156847</td>\n",
       "      <td>0.942993</td>\n",
       "      <td>0.999477</td>\n",
       "      <td>0.988458</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>74259 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            dnn        rf      lgbm       xgb  label\n",
       "0      0.227917  0.991146  0.999975  0.996300    1.0\n",
       "1      0.240160  0.991146  0.999979  0.996300    1.0\n",
       "2      0.250123  0.991146  0.999971  0.996300    1.0\n",
       "3      0.116905  0.563712  0.994721  0.951862    2.0\n",
       "4      0.108086  0.828218  0.999755  0.977432    0.0\n",
       "...         ...       ...       ...       ...    ...\n",
       "74254  0.239538  0.991146  0.999975  0.996300    1.0\n",
       "74255  0.161583  0.958180  0.999924  0.989533    1.0\n",
       "74256  0.264320  0.974195  0.999806  0.991595    0.0\n",
       "74257  0.346290  0.466003  0.897302  0.734670    0.0\n",
       "74258  0.156847  0.942993  0.999477  0.988458    1.0\n",
       "\n",
       "[74259 rows x 5 columns]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_level_00_1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dnn</th>\n",
       "      <th>rf</th>\n",
       "      <th>lgbm</th>\n",
       "      <th>xgb</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74254</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74255</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74256</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74257</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74258</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>74259 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       dnn   rf  lgbm  xgb  label\n",
       "0      0.0  1.0   1.0  1.0    1.0\n",
       "1      0.0  1.0   1.0  1.0    1.0\n",
       "2      1.0  1.0   1.0  1.0    1.0\n",
       "3      1.0  2.0   2.0  2.0    2.0\n",
       "4      3.0  0.0   0.0  0.0    0.0\n",
       "...    ...  ...   ...  ...    ...\n",
       "74254  1.0  1.0   1.0  1.0    1.0\n",
       "74255  1.0  1.0   1.0  1.0    1.0\n",
       "74256  0.0  0.0   0.0  0.0    0.0\n",
       "74257  0.0  0.0   0.0  0.0    0.0\n",
       "74258  0.0  1.0   1.0  1.0    1.0\n",
       "\n",
       "[74259 rows x 5 columns]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_level_00_0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "if pick_prob == 1:\n",
    "    df_level_01 = df_level_00_1\n",
    "else: \n",
    "    df_level_01 = df_level_00_0\n",
    "\n",
    "\n",
    "y_01 = df_level_01.pop('label')\n",
    "X_01 = df_level_01\n",
    "df_level_01 = df_level_01.assign(label = y_01)\n",
    "\n",
    "\n",
    "split = 0.7\n",
    "X_train_01,X_test_01, y_train_01, y_test_01 = sklearn.model_selection.train_test_split(X_01, y_01, train_size=split)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "# df_level_02 = pd.read_csv('base_models_class_feature_selection.csv')\n",
    "\n",
    "# df_level_02\n",
    "\n",
    "# y_02 = df_level_02.pop('label')\n",
    "# X_02 = df_level_02\n",
    "# df_level_02 = df_level_02.assign(label = y_01)\n",
    "\n",
    "\n",
    "# split = 0.7\n",
    "# X_train_02,X_test_02, y_train_02, y_test_02 = sklearn.model_selection.train_test_split(X_02, y_02, train_size=split)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training the stronger model - STACK level 01"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dnn</th>\n",
       "      <th>rf</th>\n",
       "      <th>lgbm</th>\n",
       "      <th>xgb</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>7603</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6989</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45131</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17654</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44943</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62449</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37605</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10592</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53255</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30562</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>22278 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       dnn   rf  lgbm  xgb\n",
       "7603   0.0  0.0   0.0  0.0\n",
       "6989   0.0  0.0   0.0  0.0\n",
       "45131  1.0  1.0   1.0  1.0\n",
       "17654  1.0  1.0   1.0  1.0\n",
       "44943  1.0  2.0   2.0  2.0\n",
       "...    ...  ...   ...  ...\n",
       "62449  1.0  1.0   1.0  1.0\n",
       "37605  1.0  1.0   1.0  1.0\n",
       "10592  0.0  0.0   0.0  0.0\n",
       "53255  0.0  0.0   3.0  3.0\n",
       "30562  1.0  2.0   0.0  0.0\n",
       "\n",
       "[22278 rows x 4 columns]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test_01"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Voting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       dnn   rf  lgbm  xgb  ensemble\n",
      "7603   0.0  0.0   0.0  0.0         0\n",
      "6989   0.0  0.0   0.0  0.0         0\n",
      "45131  1.0  1.0   1.0  1.0         1\n",
      "17654  1.0  1.0   1.0  1.0         1\n",
      "44943  1.0  2.0   2.0  2.0         2\n",
      "...    ...  ...   ...  ...       ...\n",
      "62449  1.0  1.0   1.0  1.0         1\n",
      "37605  1.0  1.0   1.0  1.0         1\n",
      "10592  0.0  0.0   0.0  0.0         0\n",
      "53255  0.0  0.0   3.0  3.0         0\n",
      "30562  1.0  2.0   0.0  0.0         0\n",
      "\n",
      "[22278 rows x 5 columns]\n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0.0     1.0     2.0    3.0  4.0\n",
      "0.0  11274.0    62.0    73.0   19.0  0.0\n",
      "1.0    524.0  7567.0    17.0    3.0  0.0\n",
      "2.0    157.0   175.0  1819.0    0.0  0.0\n",
      "3.0    456.0     1.0     6.0  111.0  0.0\n",
      "4.0     12.0     0.0     0.0    2.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9323547894784092\n",
      "Precision total:  0.7298217327196387\n",
      "Recall total:  0.5916975779625286\n",
      "F1 total:  0.620818025466035\n",
      "BACC total:  0.5916975779625286\n",
      "MCC total:  0.8854278984189767\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    }
   ],
   "source": [
    "if pick_prob == 0:\n",
    "    # Voting start\n",
    "\n",
    "    import pandas as pd\n",
    "    from scipy.stats import mode\n",
    "\n",
    "    # Assuming 'df' is your original DataFrame with columns 'dnn', 'rf', 'lgbm', 'ada', 'knn', 'mlp', 'svm', 'label'\n",
    "    df = X_test_01\n",
    "    # Extract predictions columns\n",
    "    \n",
    "    # predictions = df[['dnn', 'rf', 'lgbm', 'ada', 'knn', 'mlp', 'svm','cat','xgb']]\n",
    "        # selected_columns = df.loc[:, ~df.columns.isin(['rf'])]\n",
    "    predictions = df.loc[:, ~df.columns.isin(['label'])] #df[column_features]\n",
    "\n",
    "    # Use the mode function along axis 1 to get the most common prediction for each row\n",
    "    ensemble_predictions, _ = mode(predictions.values, axis=1)\n",
    "\n",
    "    # Add the ensemble predictions to the DataFrame\n",
    "    df['ensemble'] = ensemble_predictions.astype(int)\n",
    "\n",
    "    # Display the DataFrame with ensemble predictions\n",
    "    print(df)\n",
    "\n",
    "    pred_label = df ['ensemble'].values\n",
    "    df.pop('ensemble')\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Voting', file = f)\n",
    "\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('CONFUSION MATRIX')\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "    # pred_label = label[ypred]\n",
    "\n",
    "    confusion_matrix = pd.crosstab(y_test_01, pred_label,rownames=['Actual ALERT'],colnames = ['Predicted ALERT'], dropna=False).sort_index(axis=0).sort_index(axis=1)\n",
    "    all_unique_values = sorted(set(pred_label) | set(y_test_01))\n",
    "    z = np.zeros((len(all_unique_values), len(all_unique_values)))\n",
    "    rows, cols = confusion_matrix.shape\n",
    "    z[:rows, :cols] = confusion_matrix\n",
    "    confusion_matrix  = pd.DataFrame(z, columns=all_unique_values, index=all_unique_values)\n",
    "    # confusion_matrix.to_csv('Ensemble_conf_matrix.csv')\n",
    "    # with open(output_file_name, \"a\") as f:print(confusion_matrix,file=f)\n",
    "    print(confusion_matrix)\n",
    "    with open(output_file_name, \"a\") as f: print('Confusion Matrix', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print(confusion_matrix, file = f)\n",
    "\n",
    "\n",
    "    FP = confusion_matrix.sum(axis=0) - np.diag(confusion_matrix)\n",
    "    FN = confusion_matrix.sum(axis=1) - np.diag(confusion_matrix)\n",
    "    TP = np.diag(confusion_matrix)\n",
    "    TN = confusion_matrix.values.sum() - (FP + FN + TP)\n",
    "    TP_total = sum(TP)\n",
    "    TN_total = sum(TN)\n",
    "    FP_total = sum(FP)\n",
    "    FN_total = sum(FN)\n",
    "\n",
    "    TP_total = np.array(TP_total,dtype=np.float64)\n",
    "    TN_total = np.array(TN_total,dtype=np.float64)\n",
    "    FP_total = np.array(FP_total,dtype=np.float64)\n",
    "    FN_total = np.array(FN_total,dtype=np.float64)\n",
    "\n",
    "\n",
    "\n",
    "    #----------------------------------------------------------------#----------------------------------------------------------------\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('METRICS')\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "    Acc = accuracy_score(y_test_01, pred_label)\n",
    "    Precision = precision_score(y_test_01, pred_label, average='macro')\n",
    "    Recall = recall_score(y_test_01, pred_label, average='macro')\n",
    "    F1 =  f1_score(y_test_01, pred_label, average='macro')\n",
    "    BACC = balanced_accuracy_score(y_test_01, pred_label)\n",
    "    MCC = matthews_corrcoef(y_test_01, pred_label)\n",
    "\n",
    "\n",
    "    voting_acc_01 = Acc\n",
    "    voting_pre_01 = Precision\n",
    "    voting_rec_01 = Recall\n",
    "    voting_f1_01 = F1\n",
    "    voting_bacc_01 = BACC\n",
    "    voting_mcc_01 = MCC\n",
    "    # with open(output_file_name, \"a\") as f:print('Accuracy total: ', Acc,file=f)\n",
    "    print('Accuracy total: ', Acc)\n",
    "    print('Precision total: ', Precision )\n",
    "    print('Recall total: ', Recall )\n",
    "    print('F1 total: ', F1 )\n",
    "    print('BACC total: ', BACC)\n",
    "    print('MCC total: ', MCC)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Accuracy total: ', Acc, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Precision total: ', Precision, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Recall total: ', Recall , file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('F1 total: ', F1, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('BACC total: ', BACC , file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('MCC total: ', MCC, file = f)\n",
    "\n",
    "   \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Average"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# len(~df.columns.isin(['label']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       dnn   rf  lgbm  xgb  results\n",
      "7603   0.0  0.0   0.0  0.0        0\n",
      "6989   0.0  0.0   0.0  0.0        0\n",
      "45131  1.0  1.0   1.0  1.0        1\n",
      "17654  1.0  1.0   1.0  1.0        1\n",
      "44943  1.0  2.0   2.0  2.0        2\n",
      "...    ...  ...   ...  ...      ...\n",
      "62449  1.0  1.0   1.0  1.0        1\n",
      "37605  1.0  1.0   1.0  1.0        1\n",
      "10592  0.0  0.0   0.0  0.0        0\n",
      "53255  0.0  0.0   3.0  3.0        2\n",
      "30562  1.0  2.0   0.0  0.0        1\n",
      "\n",
      "[22278 rows x 5 columns]\n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0.0     1.0     2.0  3.0  4.0\n",
      "0.0  10814.0   516.0    98.0  0.0  0.0\n",
      "1.0    507.0  7579.0    25.0  0.0  0.0\n",
      "2.0     61.0   203.0  1887.0  0.0  0.0\n",
      "3.0     55.0   144.0   375.0  0.0  0.0\n",
      "4.0      6.0     4.0     4.0  0.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9103151090762187\n",
      "Precision total:  0.5264499985594033\n",
      "Recall total:  0.5515897523508212\n",
      "F1 total:  0.5384866496250866\n",
      "BACC total:  0.5515897523508212\n",
      "MCC total:  0.8482880896752031\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    }
   ],
   "source": [
    "if pick_prob == 0:\n",
    "    # Average start\n",
    "\n",
    "    import pandas as pd\n",
    "    from scipy.stats import mode\n",
    "\n",
    "    # Assuming 'df' is your original DataFrame with columns 'dnn', 'rf', 'lgbm', 'ada', 'knn', 'mlp', 'svm', 'label'\n",
    "    df = X_test_01\n",
    "    # Extract predictions columns\n",
    "    \n",
    "    # predictions = df[['dnn', 'rf', 'lgbm', 'ada', 'knn', 'mlp', 'svm','cat','xgb']]\n",
    "        # selected_columns = df.loc[:, ~df.columns.isin(['rf'])]\n",
    "    predictions = df.loc[:, ~df.columns.isin(['label'])] #df[column_features]\n",
    "\n",
    "   \n",
    "\n",
    "    column_sums = df.sum(axis=1)\n",
    "    # print(column_sums)\n",
    "    # df['column_sums'] = column_sums\n",
    "    # print(len(~df.columns.isin(['label'])))\n",
    "    \n",
    "       # Calculate the average sum along axis 1\n",
    "    row_average = df.mean(axis=1)\n",
    "\n",
    "    # Approximate the result to the closest integer\n",
    "    rounded_average = row_average.round().astype(int)\n",
    "\n",
    "    # print(rounded_average)\n",
    "\n",
    "    df['results'] = rounded_average\n",
    "    print(df)\n",
    "    # Use the mode function along axis 1 to get the most common prediction for each row\n",
    "    # ensemble_predictions, _ = mode(predictions.values, axis=1)\n",
    "\n",
    "    # Add the ensemble predictions to the DataFrame\n",
    "    # df['ensemble'] = ensemble_predictions.astype(int)/len(~df.columns.isin(['label']))\n",
    "    # df['ensemble'] = column_sums / len(~df.columns.isin(['label']))\n",
    "\n",
    " \n",
    "\n",
    "    #rules\n",
    "\n",
    "    # for i in range(0,len(df)):\n",
    "    #     if \n",
    "\n",
    "\n",
    "    # print()\n",
    "\n",
    "    # Display the DataFrame with ensemble predictions\n",
    " \n",
    "    pred_label = df ['results'].values\n",
    "\n",
    "    # pred_label = df ['ensemble'].values\n",
    "    # df.pop('ensemble')\n",
    "    df.pop('results')\n",
    "\n",
    "    # df.pop('column_sums')\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Average', file = f)\n",
    "\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('CONFUSION MATRIX')\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "    # pred_label = label[ypred]\n",
    "\n",
    "    confusion_matrix = pd.crosstab(y_test_01, pred_label,rownames=['Actual ALERT'],colnames = ['Predicted ALERT'], dropna=False).sort_index(axis=0).sort_index(axis=1)\n",
    "    all_unique_values = sorted(set(pred_label) | set(y_test_01))\n",
    "    z = np.zeros((len(all_unique_values), len(all_unique_values)))\n",
    "    rows, cols = confusion_matrix.shape\n",
    "    z[:rows, :cols] = confusion_matrix\n",
    "    confusion_matrix  = pd.DataFrame(z, columns=all_unique_values, index=all_unique_values)\n",
    "    # confusion_matrix.to_csv('Ensemble_conf_matrix.csv')\n",
    "    # with open(output_file_name, \"a\") as f:print(confusion_matrix,file=f)\n",
    "    print(confusion_matrix)\n",
    "    with open(output_file_name, \"a\") as f: print('Confusion Matrix', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print(confusion_matrix, file = f)\n",
    "\n",
    "\n",
    "    FP = confusion_matrix.sum(axis=0) - np.diag(confusion_matrix)\n",
    "    FN = confusion_matrix.sum(axis=1) - np.diag(confusion_matrix)\n",
    "    TP = np.diag(confusion_matrix)\n",
    "    TN = confusion_matrix.values.sum() - (FP + FN + TP)\n",
    "    TP_total = sum(TP)\n",
    "    TN_total = sum(TN)\n",
    "    FP_total = sum(FP)\n",
    "    FN_total = sum(FN)\n",
    "\n",
    "    TP_total = np.array(TP_total,dtype=np.float64)\n",
    "    TN_total = np.array(TN_total,dtype=np.float64)\n",
    "    FP_total = np.array(FP_total,dtype=np.float64)\n",
    "    FN_total = np.array(FN_total,dtype=np.float64)\n",
    "\n",
    "\n",
    "\n",
    "    #----------------------------------------------------------------#----------------------------------------------------------------\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('METRICS')\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "    Acc = accuracy_score(y_test_01, pred_label)\n",
    "    Precision = precision_score(y_test_01, pred_label, average='macro')\n",
    "    Recall = recall_score(y_test_01, pred_label, average='macro')\n",
    "    F1 =  f1_score(y_test_01, pred_label, average='macro')\n",
    "    BACC = balanced_accuracy_score(y_test_01, pred_label)\n",
    "    MCC = matthews_corrcoef(y_test_01, pred_label)\n",
    "\n",
    "\n",
    "    avg_acc_01 = Acc\n",
    "    avg_pre_01 = Precision\n",
    "    avg_rec_01 = Recall\n",
    "    avg_f1_01 = F1\n",
    "    avg_bacc_01 = BACC\n",
    "    avg_mcc_01 = MCC\n",
    "    # with open(output_file_name, \"a\") as f:print('Accuracy total: ', Acc,file=f)\n",
    "    print('Accuracy total: ', Acc)\n",
    "    print('Precision total: ', Precision )\n",
    "    print('Recall total: ', Recall )\n",
    "    print('F1 total: ', F1 )\n",
    "    print('BACC total: ', BACC)\n",
    "    print('MCC total: ', MCC)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Accuracy total: ', Acc, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Precision total: ', Precision, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Recall total: ', Recall , file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('F1 total: ', F1, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('BACC total: ', BACC , file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('MCC total: ', MCC, file = f)\n",
    "\n",
    "   \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Weighed Average"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import pandas as pd\n",
    "\n",
    "# # Assuming df is your DataFrame\n",
    "# data = {'dnn': [1.0, 0.0], 'rf': [1.0, 0.0], 'lgbm': [1.0, 0.0], 'xgb': [0.0, 0.0]}\n",
    "# weights = {'dnn': 0.4, 'rf': 0.3, 'lgbm': 0.2, 'xgb': 0.1}\n",
    "# df = pd.DataFrame(data)\n",
    "\n",
    "# # Calculate the weighted average along axis 1\n",
    "# weighted_average = (df.multiply(weights)).sum(axis=1) / sum(weights.values())\n",
    "\n",
    "# # Approximate the result to the closest integer\n",
    "# rounded_weighted_average = weighted_average.round().astype(int)\n",
    "\n",
    "# print(rounded_weighted_average)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# selected_col = ['xgb','lgbm']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df[['rf', 'dnn','lgbm']]\n",
    "# # df[['rf', 'dnn']]\n",
    "# df = df[selected_col]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>xgb</th>\n",
       "      <th>lgbm</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>7603</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6989</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45131</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17654</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44943</th>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62449</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37605</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10592</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53255</th>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30562</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>22278 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       xgb  lgbm\n",
       "7603   0.0   0.0\n",
       "6989   0.0   0.0\n",
       "45131  1.0   1.0\n",
       "17654  1.0   1.0\n",
       "44943  2.0   2.0\n",
       "...    ...   ...\n",
       "62449  1.0   1.0\n",
       "37605  1.0   1.0\n",
       "10592  0.0   0.0\n",
       "53255  3.0   3.0\n",
       "30562  0.0   0.0\n",
       "\n",
       "[22278 rows x 2 columns]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>xgb</th>\n",
       "      <th>lgbm</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>7603</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6989</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45131</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17654</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44943</th>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62449</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37605</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10592</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53255</th>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30562</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>22278 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       xgb  lgbm\n",
       "7603   0.0   0.0\n",
       "6989   0.0   0.0\n",
       "45131  1.0   1.0\n",
       "17654  1.0   1.0\n",
       "44943  2.0   2.0\n",
       "...    ...   ...\n",
       "62449  1.0   1.0\n",
       "37605  1.0   1.0\n",
       "10592  0.0   0.0\n",
       "53255  3.0   3.0\n",
       "30562  0.0   0.0\n",
       "\n",
       "[22278 rows x 2 columns]"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# column_features\n",
    "# move this up with column_features\n",
    "\n",
    "#important update this as you need to select the important features,\n",
    "#  the left is the least important while the right is the most important\n",
    "# needs automation\n",
    "feature_selection_columns_in_order_of_importance = [ 'xgb','lgbm', 'rf', 'dnn']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.0, 0.3333333333333333, 0.6666666666666666, 1.0]\n",
      "[0.0, 0.3333333333333333, 0.6666666666666666, 1.0]\n",
      "       xgb  lgbm   rf  dnn\n",
      "7603   0.0   0.0  0.0  0.0\n",
      "6989   0.0   0.0  0.0  0.0\n",
      "45131  1.0   1.0  1.0  1.0\n",
      "17654  1.0   1.0  1.0  1.0\n",
      "44943  2.0   2.0  2.0  1.0\n",
      "...    ...   ...  ...  ...\n",
      "62449  1.0   1.0  1.0  1.0\n",
      "37605  1.0   1.0  1.0  1.0\n",
      "10592  0.0   0.0  0.0  0.0\n",
      "53255  3.0   3.0  0.0  0.0\n",
      "30562  0.0   0.0  2.0  1.0\n",
      "\n",
      "[22278 rows x 4 columns]\n",
      "7603     0.000000\n",
      "6989     0.000000\n",
      "45131    1.000000\n",
      "17654    1.000000\n",
      "44943    1.500000\n",
      "           ...   \n",
      "62449    1.000000\n",
      "37605    1.000000\n",
      "10592    0.000000\n",
      "53255    0.500000\n",
      "30562    1.166667\n",
      "Length: 22278, dtype: float64\n",
      "7603     0\n",
      "6989     0\n",
      "45131    1\n",
      "17654    1\n",
      "44943    2\n",
      "        ..\n",
      "62449    1\n",
      "37605    1\n",
      "10592    0\n",
      "53255    0\n",
      "30562    1\n",
      "Length: 22278, dtype: int64\n",
      "       xgb  lgbm   rf  dnn  results\n",
      "7603   0.0   0.0  0.0  0.0        0\n",
      "6989   0.0   0.0  0.0  0.0        0\n",
      "45131  1.0   1.0  1.0  1.0        1\n",
      "17654  1.0   1.0  1.0  1.0        1\n",
      "44943  2.0   2.0  2.0  1.0        2\n",
      "...    ...   ...  ...  ...      ...\n",
      "62449  1.0   1.0  1.0  1.0        1\n",
      "37605  1.0   1.0  1.0  1.0        1\n",
      "10592  0.0   0.0  0.0  0.0        0\n",
      "53255  3.0   3.0  0.0  0.0        2\n",
      "30562  0.0   0.0  2.0  1.0        1\n",
      "\n",
      "[22278 rows x 5 columns]\n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0.0     1.0     2.0  3.0  4.0\n",
      "0.0  10814.0   516.0    98.0  0.0  0.0\n",
      "1.0    507.0  7579.0    25.0  0.0  0.0\n",
      "2.0     61.0   203.0  1887.0  0.0  0.0\n",
      "3.0     55.0   144.0   375.0  0.0  0.0\n",
      "4.0      6.0     4.0     4.0  0.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9103151090762187\n",
      "Precision total:  0.5264499985594033\n",
      "Recall total:  0.5515897523508212\n",
      "F1 total:  0.5384866496250866\n",
      "BACC total:  0.5515897523508212\n",
      "MCC total:  0.8482880896752031\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    }
   ],
   "source": [
    "if pick_prob == 0:\n",
    "    # Average start\n",
    "\n",
    "    import pandas as pd\n",
    "    from scipy.stats import mode\n",
    "\n",
    "    # Assuming 'df' is your original DataFrame with columns 'dnn', 'rf', 'lgbm', 'ada', 'knn', 'mlp', 'svm', 'label'\n",
    "    # df = X_test_01\n",
    "    df = X_test_01[feature_selection_columns_in_order_of_importance]\n",
    "    # Extract predictions columns\n",
    "    \n",
    "    # predictions = df[['dnn', 'rf', 'lgbm', 'ada', 'knn', 'mlp', 'svm','cat','xgb']]\n",
    "        # selected_columns = df.loc[:, ~df.columns.isin(['rf'])]\n",
    "    predictions = df.loc[:, ~df.columns.isin(['label'])] #df[column_features]\n",
    "\n",
    "    # weight\n",
    "    weights_values = []\n",
    "\n",
    "    # linear weight distribution\n",
    "    for i in range(0,len(~df.columns.isin(['label']))):\n",
    "        weights_values.append(i/(len(~df.columns.isin(['label']))-1))\n",
    "    print(weights_values)\n",
    "    # weights_values = [10,3,2,2.3]\n",
    "    print(weights_values)\n",
    "    print(df)\n",
    "    weighted_average = df.multiply(weights_values).sum(axis=1) / sum(weights_values)\n",
    "    print(weighted_average)\n",
    "\n",
    "    # weights = {'dnn': 0.4, 'rf': 0.3, 'lgbm': 0.2, 'xgb': 0.1}\n",
    "\n",
    "\n",
    "    # Calculate the weighted average along axis 1\n",
    "    # weighted_average = (df.multiply(weights)).sum(axis=1) / sum(weights.values())\n",
    "\n",
    "    # Approximate the result to the closest integer\n",
    "    rounded_weighted_average = weighted_average.round().astype(int)\n",
    "\n",
    "    print(rounded_weighted_average)\n",
    "\n",
    "    # print(rounded_average)\n",
    "\n",
    "    df['results'] = rounded_average\n",
    "    print(df)\n",
    "    # Use the mode function along axis 1 to get the most common prediction for each row\n",
    "    # ensemble_predictions, _ = mode(predictions.values, axis=1)\n",
    "\n",
    "    # Add the ensemble predictions to the DataFrame\n",
    "    # df['ensemble'] = ensemble_predictions.astype(int)/len(~df.columns.isin(['label']))\n",
    "    # df['ensemble'] = column_sums / len(~df.columns.isin(['label']))\n",
    "\n",
    " \n",
    "\n",
    "    #rules\n",
    "\n",
    "    # for i in range(0,len(df)):\n",
    "    #     if \n",
    "\n",
    "\n",
    "    # print()\n",
    "\n",
    "    # Display the DataFrame with ensemble predictions\n",
    " \n",
    "    pred_label = df ['results'].values\n",
    "\n",
    "    # pred_label = df ['ensemble'].values\n",
    "    # df.pop('ensemble')\n",
    "    df.pop('results')\n",
    "\n",
    "    # df.pop('column_sums')\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Weighed Average', file = f)\n",
    "\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('CONFUSION MATRIX')\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "    # pred_label = label[ypred]\n",
    "\n",
    "    confusion_matrix = pd.crosstab(y_test_01, pred_label,rownames=['Actual ALERT'],colnames = ['Predicted ALERT'], dropna=False).sort_index(axis=0).sort_index(axis=1)\n",
    "    all_unique_values = sorted(set(pred_label) | set(y_test_01))\n",
    "    z = np.zeros((len(all_unique_values), len(all_unique_values)))\n",
    "    rows, cols = confusion_matrix.shape\n",
    "    z[:rows, :cols] = confusion_matrix\n",
    "    confusion_matrix  = pd.DataFrame(z, columns=all_unique_values, index=all_unique_values)\n",
    "    # confusion_matrix.to_csv('Ensemble_conf_matrix.csv')\n",
    "    # with open(output_file_name, \"a\") as f:print(confusion_matrix,file=f)\n",
    "    print(confusion_matrix)\n",
    "    with open(output_file_name, \"a\") as f: print('Confusion Matrix', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print(confusion_matrix, file = f)\n",
    "\n",
    "\n",
    "    FP = confusion_matrix.sum(axis=0) - np.diag(confusion_matrix)\n",
    "    FN = confusion_matrix.sum(axis=1) - np.diag(confusion_matrix)\n",
    "    TP = np.diag(confusion_matrix)\n",
    "    TN = confusion_matrix.values.sum() - (FP + FN + TP)\n",
    "    TP_total = sum(TP)\n",
    "    TN_total = sum(TN)\n",
    "    FP_total = sum(FP)\n",
    "    FN_total = sum(FN)\n",
    "\n",
    "    TP_total = np.array(TP_total,dtype=np.float64)\n",
    "    TN_total = np.array(TN_total,dtype=np.float64)\n",
    "    FP_total = np.array(FP_total,dtype=np.float64)\n",
    "    FN_total = np.array(FN_total,dtype=np.float64)\n",
    "\n",
    "\n",
    "\n",
    "    #----------------------------------------------------------------#----------------------------------------------------------------\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('METRICS')\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "    Acc = accuracy_score(y_test_01, pred_label)\n",
    "    Precision = precision_score(y_test_01, pred_label, average='macro')\n",
    "    Recall = recall_score(y_test_01, pred_label, average='macro')\n",
    "    F1 =  f1_score(y_test_01, pred_label, average='macro')\n",
    "    BACC = balanced_accuracy_score(y_test_01, pred_label)\n",
    "    MCC = matthews_corrcoef(y_test_01, pred_label)\n",
    "\n",
    "\n",
    "    weighed_avg_acc_01 = Acc\n",
    "    weighed_avg_pre_01 = Precision\n",
    "    weighed_avg_rec_01 = Recall\n",
    "    weighed_avg_f1_01 = F1\n",
    "    weighed_avg_bacc_01 = BACC\n",
    "    weighed_avg_mcc_01 = MCC\n",
    "    # with open(output_file_name, \"a\") as f:print('Accuracy total: ', Acc,file=f)\n",
    "    print('Accuracy total: ', Acc)\n",
    "    print('Precision total: ', Precision )\n",
    "    print('Recall total: ', Recall )\n",
    "    print('F1 total: ', F1 )\n",
    "    print('BACC total: ', BACC)\n",
    "    print('MCC total: ', MCC)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Accuracy total: ', Acc, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Precision total: ', Precision, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Recall total: ', Recall , file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('F1 total: ', F1, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('BACC total: ', BACC , file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('MCC total: ', MCC, file = f)\n",
    "\n",
    "   \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## bagging  with DT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0.0     1.0     2.0    3.0  4.0\n",
      "0.0  11214.0    69.0    54.0   91.0  0.0\n",
      "1.0     51.0  8052.0     4.0    4.0  0.0\n",
      "2.0     45.0     6.0  2096.0    4.0  0.0\n",
      "3.0     66.0     1.0     8.0  499.0  0.0\n",
      "4.0     10.0     0.0     0.0    4.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9812819822246163\n",
      "Precision total:  0.7547839400757195\n",
      "Recall total:  0.7635536935985494\n",
      "F1 total:  0.7590703964537397\n",
      "BACC total:  0.7635536935985494\n",
      "MCC total:  0.9685429694278876\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import BaggingClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.datasets import load_iris\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "# # Load the Iris dataset\n",
    "# iris = load_iris()\n",
    "# X, y = iris.data, iris.target\n",
    "\n",
    "# # Split the data into training and testing sets\n",
    "# X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n",
    "\n",
    "# # Define the base classifier (Decision Tree in this case)\n",
    "base_classifier = DecisionTreeClassifier(random_state=42)\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train_01, y_train_01)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test_01)\n",
    "\n",
    "# Evaluate accuracy\n",
    "# accuracy = accuracy_score(y_test_01, y_pred)\n",
    "# print(f'Accuracy: {accuracy}')\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('Bagging with DT', file = f)\n",
    "\n",
    "\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('CONFUSION MATRIX')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "pred_label = y_pred\n",
    "\n",
    "confusion_matrix = pd.crosstab(y_test_01, pred_label,rownames=['Actual ALERT'],colnames = ['Predicted ALERT'], dropna=False).sort_index(axis=0).sort_index(axis=1)\n",
    "all_unique_values = sorted(set(pred_label) | set(y_test_01))\n",
    "z = np.zeros((len(all_unique_values), len(all_unique_values)))\n",
    "rows, cols = confusion_matrix.shape\n",
    "z[:rows, :cols] = confusion_matrix\n",
    "confusion_matrix  = pd.DataFrame(z, columns=all_unique_values, index=all_unique_values)\n",
    "# confusion_matrix.to_csv('Ensemble_conf_matrix.csv')\n",
    "# with open(output_file_name, \"a\") as f:print(confusion_matrix,file=f)\n",
    "print(confusion_matrix)\n",
    "with open(output_file_name, \"a\") as f: print('Confusion Matrix', file = f)\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print(confusion_matrix, file = f)\n",
    "\n",
    "\n",
    "FP = confusion_matrix.sum(axis=0) - np.diag(confusion_matrix)\n",
    "FN = confusion_matrix.sum(axis=1) - np.diag(confusion_matrix)\n",
    "TP = np.diag(confusion_matrix)\n",
    "TN = confusion_matrix.values.sum() - (FP + FN + TP)\n",
    "TP_total = sum(TP)\n",
    "TN_total = sum(TN)\n",
    "FP_total = sum(FP)\n",
    "FN_total = sum(FN)\n",
    "\n",
    "TP_total = np.array(TP_total,dtype=np.float64)\n",
    "TN_total = np.array(TN_total,dtype=np.float64)\n",
    "FP_total = np.array(FP_total,dtype=np.float64)\n",
    "FN_total = np.array(FN_total,dtype=np.float64)\n",
    "\n",
    "\n",
    "\n",
    "#----------------------------------------------------------------#----------------------------------------------------------------\n",
    "\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('METRICS')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "Acc = accuracy_score(y_test_01, pred_label)\n",
    "Precision = precision_score(y_test_01, pred_label, average='macro')\n",
    "Recall = recall_score(y_test_01, pred_label, average='macro')\n",
    "F1 =  f1_score(y_test_01, pred_label, average='macro')\n",
    "BACC = balanced_accuracy_score(y_test_01, pred_label)\n",
    "MCC = matthews_corrcoef(y_test_01, pred_label)\n",
    "\n",
    "\n",
    "bag_dt_acc_01 = Acc\n",
    "bag_dt_pre_01 = Precision\n",
    "bag_dt_rec_01 = Recall\n",
    "bag_dt_f1_01 = F1\n",
    "bag_dt_bacc_01 = BACC\n",
    "bag_dt_mcc_01 = MCC\n",
    "# with open(output_file_name, \"a\") as f:print('Accuracy total: ', Acc,file=f)\n",
    "print('Accuracy total: ', Acc)\n",
    "print('Precision total: ', Precision )\n",
    "print('Recall total: ', Recall )\n",
    "print('F1 total: ', F1 )\n",
    "print('BACC total: ', BACC)\n",
    "print('MCC total: ', MCC)\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('Accuracy total: ', Acc, file = f)\n",
    "with open(output_file_name, \"a\") as f: print('Precision total: ', Precision, file = f)\n",
    "with open(output_file_name, \"a\") as f: print('Recall total: ', Recall , file = f)\n",
    "with open(output_file_name, \"a\") as f: print('F1 total: ', F1, file = f)\n",
    "with open(output_file_name, \"a\") as f: print('BACC total: ', BACC , file = f)\n",
    "with open(output_file_name, \"a\") as f: print('MCC total: ', MCC, file = f)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## bagging  with SVM\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0.0     1.0    2.0  3.0  4.0\n",
      "0.0  11094.0   223.0  111.0  0.0  0.0\n",
      "1.0    522.0  7583.0    6.0  0.0  0.0\n",
      "2.0     31.0  1950.0  170.0  0.0  0.0\n",
      "3.0     63.0    29.0  482.0  0.0  0.0\n",
      "4.0      8.0     2.0    4.0  0.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.8599964090133764\n",
      "Precision total:  0.4690193067460854\n",
      "Recall total:  0.5490796021769268\n",
      "F1 total:  0.5043264490349578\n",
      "BACC total:  0.5490796021769268\n",
      "MCC total:  0.7656097087636203\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    }
   ],
   "source": [
    "## bagging  with SVM\n",
    "from sklearn.ensemble import BaggingClassifier\n",
    "# from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "\n",
    "\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "\n",
    "# Instantiate the SGDClassifier with additional hyperparameters\n",
    "svm_01 = SGDClassifier(\n",
    "    loss='hinge',           # hinge loss for linear SVM\n",
    "    penalty='l2',           # L2 regularization to prevent overfitting\n",
    "    alpha=1e-4,             # Learning rate (small value for fine-grained updates)\n",
    "    max_iter=1000,          # Number of passes over the training data\n",
    "    random_state=42,        # Seed for reproducible results\n",
    "    learning_rate='optimal' # Automatically adjusts the learning rate based on the training data\n",
    ")\n",
    "# # Load the Iris dataset\n",
    "# iris = load_iris()\n",
    "# X, y = iris.data, iris.target\n",
    "\n",
    "# # Split the data into training and testing sets\n",
    "# X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n",
    "\n",
    "# # Define the base classifier (Decision Tree in this case)\n",
    "base_classifier = svm_01\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train_01, y_train_01)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test_01)\n",
    "\n",
    "# Evaluate accuracy\n",
    "# accuracy = accuracy_score(y_test_01, y_pred)\n",
    "# print(f'Accuracy: {accuracy}')\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('Bagging with SVM', file = f)\n",
    "\n",
    "\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('CONFUSION MATRIX')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "pred_label = y_pred\n",
    "\n",
    "confusion_matrix = pd.crosstab(y_test_01, pred_label,rownames=['Actual ALERT'],colnames = ['Predicted ALERT'], dropna=False).sort_index(axis=0).sort_index(axis=1)\n",
    "all_unique_values = sorted(set(pred_label) | set(y_test_01))\n",
    "z = np.zeros((len(all_unique_values), len(all_unique_values)))\n",
    "rows, cols = confusion_matrix.shape\n",
    "z[:rows, :cols] = confusion_matrix\n",
    "confusion_matrix  = pd.DataFrame(z, columns=all_unique_values, index=all_unique_values)\n",
    "# confusion_matrix.to_csv('Ensemble_conf_matrix.csv')\n",
    "# with open(output_file_name, \"a\") as f:print(confusion_matrix,file=f)\n",
    "print(confusion_matrix)\n",
    "with open(output_file_name, \"a\") as f: print('Confusion Matrix', file = f)\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print(confusion_matrix, file = f)\n",
    "\n",
    "\n",
    "FP = confusion_matrix.sum(axis=0) - np.diag(confusion_matrix)\n",
    "FN = confusion_matrix.sum(axis=1) - np.diag(confusion_matrix)\n",
    "TP = np.diag(confusion_matrix)\n",
    "TN = confusion_matrix.values.sum() - (FP + FN + TP)\n",
    "TP_total = sum(TP)\n",
    "TN_total = sum(TN)\n",
    "FP_total = sum(FP)\n",
    "FN_total = sum(FN)\n",
    "\n",
    "TP_total = np.array(TP_total,dtype=np.float64)\n",
    "TN_total = np.array(TN_total,dtype=np.float64)\n",
    "FP_total = np.array(FP_total,dtype=np.float64)\n",
    "FN_total = np.array(FN_total,dtype=np.float64)\n",
    "\n",
    "\n",
    "\n",
    "#----------------------------------------------------------------#----------------------------------------------------------------\n",
    "\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('METRICS')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "Acc = accuracy_score(y_test_01, pred_label)\n",
    "Precision = precision_score(y_test_01, pred_label, average='macro')\n",
    "Recall = recall_score(y_test_01, pred_label, average='macro')\n",
    "F1 =  f1_score(y_test_01, pred_label, average='macro')\n",
    "BACC = balanced_accuracy_score(y_test_01, pred_label)\n",
    "MCC = matthews_corrcoef(y_test_01, pred_label)\n",
    "\n",
    "\n",
    "bag_svm_acc_01 = Acc\n",
    "bag_svm_pre_01 = Precision\n",
    "bag_svm_rec_01 = Recall\n",
    "bag_svm_f1_01 = F1\n",
    "bag_svm_bacc_01 = BACC\n",
    "bag_svm_mcc_01 = MCC\n",
    "# with open(output_file_name, \"a\") as f:print('Accuracy total: ', Acc,file=f)\n",
    "print('Accuracy total: ', Acc)\n",
    "print('Precision total: ', Precision )\n",
    "print('Recall total: ', Recall )\n",
    "print('F1 total: ', F1 )\n",
    "print('BACC total: ', BACC)\n",
    "print('MCC total: ', MCC)\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('Accuracy total: ', Acc, file = f)\n",
    "with open(output_file_name, \"a\") as f: print('Precision total: ', Precision, file = f)\n",
    "with open(output_file_name, \"a\") as f: print('Recall total: ', Recall , file = f)\n",
    "with open(output_file_name, \"a\") as f: print('F1 total: ', F1, file = f)\n",
    "with open(output_file_name, \"a\") as f: print('BACC total: ', BACC , file = f)\n",
    "with open(output_file_name, \"a\") as f: print('MCC total: ', MCC, file = f)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### DNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Defining DNN Model\n",
      "---------------------------------------------------------------------------------\n",
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense (Dense)                (None, 3)                 15        \n",
      "_________________________________________________________________\n",
      "dropout (Dropout)            (None, 3)                 0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 3)                 12        \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 3)                 0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 3)                 12        \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 3)                 0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 3)                 12        \n",
      "_________________________________________________________________\n",
      "dropout_3 (Dropout)          (None, 3)                 0         \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 3)                 12        \n",
      "_________________________________________________________________\n",
      "dropout_4 (Dropout)          (None, 3)                 0         \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 5)                 20        \n",
      "=================================================================\n",
      "Total params: 83\n",
      "Trainable params: 83\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining DNN Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense\n",
    "\n",
    "#Model Parameters\n",
    "dropout_rate = 0.2\n",
    "nodes = 3\n",
    "out_layer = 5\n",
    "optimizer='adam'\n",
    "loss='sparse_categorical_crossentropy'\n",
    "epochs=100\n",
    "batch_size=128\n",
    "\n",
    "\n",
    "num_columns = X_train_01.shape[1]\n",
    "\n",
    "dnn_01 = tf.keras.Sequential()\n",
    "\n",
    "# Input layer\n",
    "dnn_01.add(tf.keras.Input(shape=(num_columns,)))\n",
    "\n",
    "# # Dense layers with dropout\n",
    "# dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "# dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "# dnn_01.add(tf.keras.layers.Dense(2*nodes))\n",
    "# dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "# dnn_01.add(tf.keras.layers.Dense(3*nodes))\n",
    "# dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "# dnn_01.add(tf.keras.layers.Dense(2*nodes))\n",
    "# dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "# dnn.add(tf.keras.layers.Dense(nodes))\n",
    "# dnn.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "\n",
    "\n",
    "# Dense layers with dropout\n",
    "dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "# Output layer\n",
    "# dnn_01.add(tf.keras.layers.Dense(out_layer))\n",
    "\n",
    "dnn_01.add(tf.keras.layers.Dense(out_layer, activation='softmax'))\n",
    "\n",
    "\n",
    "dnn_01.compile(optimizer=optimizer, loss=loss,metrics=['accuracy'])\n",
    "\n",
    "dnn_01.summary()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# X_train_02 = X_train_01"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "# X_train_02 = X_train_02.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "# X_train_03 = pd.DataFrame(X_train_02) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Create a mask to filter out non-numeric values\n",
    "# mask = np.char.isnumeric(X_train_02)\n",
    "\n",
    "# # Apply the mask to the original array\n",
    "# arr_numeric = np.where(mask, X_train_02, np.nan).astype(float)\n",
    "\n",
    "# print(arr_numeric)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "# X_train_02\n",
    "# X_train_02 = X_train_02.astype(float)\n",
    "\n",
    "# print(X_train_02)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "# X_train_02 = X_train_01\n",
    "# X_train_02.columns = None\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Training DNN\n",
      "---------------------------------------------------------------------------------\n",
      "Epoch 1/100\n",
      "325/325 [==============================] - 1s 2ms/step - loss: 1.3347 - accuracy: 0.5873 - val_loss: 0.7099 - val_accuracy: 0.8708\n",
      "Epoch 2/100\n",
      "325/325 [==============================] - 1s 2ms/step - loss: 0.8427 - accuracy: 0.7290 - val_loss: 0.4766 - val_accuracy: 0.8578\n",
      "Epoch 3/100\n",
      "325/325 [==============================] - 1s 2ms/step - loss: 0.7015 - accuracy: 0.7833 - val_loss: 0.4180 - val_accuracy: 0.8572\n",
      "Epoch 4/100\n",
      "325/325 [==============================] - 1s 2ms/step - loss: 0.6381 - accuracy: 0.8062 - val_loss: 0.3873 - val_accuracy: 0.9292\n",
      "Epoch 5/100\n",
      "325/325 [==============================] - 1s 2ms/step - loss: 0.5971 - accuracy: 0.8254 - val_loss: 0.3836 - val_accuracy: 0.9291\n",
      "Epoch 6/100\n",
      "325/325 [==============================] - 1s 2ms/step - loss: 0.5660 - accuracy: 0.8408 - val_loss: 0.3695 - val_accuracy: 0.9437\n",
      "Epoch 7/100\n",
      "285/325 [=========================>....] - ETA: 0s - loss: 0.5575 - accuracy: 0.8476"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "325/325 [==============================] - 1s 2ms/step - loss: 0.5571 - accuracy: 0.8475 - val_loss: 0.3642 - val_accuracy: 0.9362\n",
      "Epoch 8/100\n",
      "325/325 [==============================] - 1s 2ms/step - loss: 0.5390 - accuracy: 0.8539 - val_loss: 0.3567 - val_accuracy: 0.9417\n",
      "Epoch 9/100\n",
      "325/325 [==============================] - 1s 2ms/step - loss: 0.5327 - accuracy: 0.8571 - val_loss: 0.3563 - val_accuracy: 0.9351\n",
      "Epoch 10/100\n",
      "325/325 [==============================] - 1s 2ms/step - loss: 0.5210 - accuracy: 0.8618 - val_loss: 0.3533 - val_accuracy: 0.9351\n",
      "Epoch 11/100\n",
      "325/325 [==============================] - 1s 2ms/step - loss: 0.5167 - accuracy: 0.8663 - val_loss: 0.3527 - val_accuracy: 0.9344\n",
      "Epoch 12/100\n",
      "325/325 [==============================] - 1s 2ms/step - loss: 0.5102 - accuracy: 0.8726 - val_loss: 0.3509 - val_accuracy: 0.9426\n",
      "Epoch 13/100\n",
      "325/325 [==============================] - 1s 2ms/step - loss: 0.5099 - accuracy: 0.8761 - val_loss: 0.3492 - val_accuracy: 0.9414\n",
      "Epoch 14/100\n",
      "325/325 [==============================] - 1s 2ms/step - loss: 0.5023 - accuracy: 0.8806 - val_loss: 0.3512 - val_accuracy: 0.9408\n",
      "Epoch 15/100\n",
      "325/325 [==============================] - 1s 2ms/step - loss: 0.5003 - accuracy: 0.8791 - val_loss: 0.3441 - val_accuracy: 0.9398\n",
      "Epoch 16/100\n",
      "325/325 [==============================] - 1s 2ms/step - loss: 0.4936 - accuracy: 0.8791 - val_loss: 0.3435 - val_accuracy: 0.9413\n"
     ]
    }
   ],
   "source": [
    "#DNN\n",
    "try:\n",
    "    from keras.callbacks import EarlyStopping\n",
    "\n",
    "    # Define EarlyStopping callback\n",
    "    early_stopping = EarlyStopping(monitor='val_accuracy', patience=10, restore_best_weights=True)\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training DNN')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Training DNN', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    # Convert Y_test back to its original format\n",
    "    # y_test = np.argmax(Y_test, axis=1)\n",
    "\n",
    "    # Start the timer\n",
    "    start = time.time()\n",
    "    # dnn_01.fit(X_train_01, y_train_01, epochs=epochs, batch_size=batch_size)\n",
    "    dnn_01.fit(X_train_01, y_train_01, epochs=epochs, batch_size=batch_size,validation_split=0.2, callbacks=[early_stopping])\n",
    "\n",
    "    # model.fit(x_train, Y_train, epochs=100, batch_size=128, validation_split=0.2, callbacks=[early_stopping])\n",
    "\n",
    "    # End the timer\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "    # joblib.dump(dnn_01, 'dnn_level_01.joblib')\n",
    "    # dnn_01.save(\"dnn_level_01.h5\")\n",
    "\n",
    "    # Calculate the time taken and print it out\n",
    "    # print(f'Time taken for training: {time_taken} seconds')\n",
    "except: \n",
    "    None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dnn_01 = load_model(\"dnn_level_01.h5\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "#DNN\n",
    "try:\n",
    "    start = time.time()\n",
    "    pred_dnn = dnn_01.predict(X_test_01)\n",
    "    preds_dnn_01 = np.argmax(pred_dnn,axis = 1)\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "except:\n",
    "        with open(output_file_name, \"a\") as f: print('error', file = f)\n",
    "        preds_dnn_01 = 0\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "# y_test = y_test_01"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "#----------------------------------------------------------------\n",
    "with open(output_file_name, \"a\") as f: print('Stack model - Strong learner - level 01', file = f)\n",
    "with open(output_file_name, \"a\") as f: print('-------------------------------------------------------', file = f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0.0     1.0     2.0  3.0  4.0\n",
      "0.0  11210.0    90.0   128.0  0.0  0.0\n",
      "1.0     59.0  8041.0    11.0  0.0  0.0\n",
      "2.0     45.0   348.0  1758.0  0.0  0.0\n",
      "3.0     65.0     4.0   505.0  0.0  0.0\n",
      "4.0     10.0     0.0     4.0  0.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9430379746835443\n",
      "Precision total:  0.5325704376912934\n",
      "Recall total:  0.5579176145445526\n",
      "F1 total:  0.5446612807463953\n",
      "BACC total:  0.5579176145445526\n",
      "MCC total:  0.9041169056517298\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('CONFUSION MATRIX')\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    with open(output_file_name, \"a\") as f: print('DNN', file = f)\n",
    "    pred_label = preds_dnn_01\n",
    "\n",
    "    # pred_label = ypred\n",
    "    #pred_label = label[ypred]\n",
    "\n",
    "    confusion_matrix = pd.crosstab(y_test_01, pred_label,rownames=['Actual ALERT'],colnames = ['Predicted ALERT'], dropna=False).sort_index(axis=0).sort_index(axis=1)\n",
    "    all_unique_values = sorted(set(pred_label) | set(y_test_01))\n",
    "    z = np.zeros((len(all_unique_values), len(all_unique_values)))\n",
    "    rows, cols = confusion_matrix.shape\n",
    "    z[:rows, :cols] = confusion_matrix\n",
    "    confusion_matrix  = pd.DataFrame(z, columns=all_unique_values, index=all_unique_values)\n",
    "    # confusion_matrix.to_csv('Ensemble_conf_matrix.csv')\n",
    "    # with open(output_file_name, \"a\") as f:print(confusion_matrix,file=f)\n",
    "    print(confusion_matrix)\n",
    "    with open(output_file_name, \"a\") as f: print('Confusion Matrix', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print(confusion_matrix, file = f)\n",
    "\n",
    "\n",
    "    FP = confusion_matrix.sum(axis=0) - np.diag(confusion_matrix)\n",
    "    FN = confusion_matrix.sum(axis=1) - np.diag(confusion_matrix)\n",
    "    TP = np.diag(confusion_matrix)\n",
    "    TN = confusion_matrix.values.sum() - (FP + FN + TP)\n",
    "    TP_total = sum(TP)\n",
    "    TN_total = sum(TN)\n",
    "    FP_total = sum(FP)\n",
    "    FN_total = sum(FN)\n",
    "\n",
    "    TP_total = np.array(TP_total,dtype=np.float64)\n",
    "    TN_total = np.array(TN_total,dtype=np.float64)\n",
    "    FP_total = np.array(FP_total,dtype=np.float64)\n",
    "    FN_total = np.array(FN_total,dtype=np.float64)\n",
    "\n",
    "\n",
    "\n",
    "    #----------------------------------------------------------------#----------------------------------------------------------------\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('METRICS')\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "    # Acc = ACC(TP_total,TN_total, FP_total, FN_total)\n",
    "    # Precision = PRECISION(TP_total, FP_total)\n",
    "    # Recall = RECALL(TP_total, FN_total)\n",
    "    # F1 = F1(Recall,Precision)\n",
    "    # BACC = BACC(TP_total,TN_total, FP_total, FN_total)\n",
    "    # MCC = MCC(TP_total,TN_total, FP_total, FN_total)\n",
    "\n",
    "\n",
    "    Acc = accuracy_score(y_test_01, pred_label)\n",
    "    Precision = precision_score(y_test_01, pred_label, average='macro')\n",
    "    Recall = recall_score(y_test_01, pred_label, average='macro')\n",
    "    F1 =  f1_score(y_test_01, pred_label, average='macro')\n",
    "    BACC = balanced_accuracy_score(y_test_01, pred_label)\n",
    "    MCC = matthews_corrcoef(y_test_01, pred_label)\n",
    "\n",
    "\n",
    "    dnn_acc_01 = Acc\n",
    "    dnn_pre_01 = Precision\n",
    "    dnn_rec_01 = Recall\n",
    "    dnn_f1_01 = F1\n",
    "    dnn_bacc_01 = BACC\n",
    "    dnn_mcc_01 = MCC\n",
    "\n",
    "    # with open(output_file_name, \"a\") as f:print('Accuracy total: ', Acc,file=f)\n",
    "    print('Accuracy total: ', Acc)\n",
    "    print('Precision total: ', Precision )\n",
    "    print('Recall total: ', Recall )\n",
    "    print('F1 total: ', F1 )\n",
    "    print('BACC total: ', BACC)\n",
    "    print('MCC total: ', MCC)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Accuracy total: ', Acc, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Precision total: ', Precision, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Recall total: ', Recall , file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('F1 total: ', F1, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('BACC total: ', BACC , file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('MCC total: ', MCC, file = f)\n",
    "except: None\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Defining SVM Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "#SVM\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining SVM Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "\n",
    "from sklearn.multioutput import MultiOutputClassifier\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "\n",
    "# Instantiate the SGDClassifier with additional hyperparameters\n",
    "clf = SGDClassifier(\n",
    "    loss='hinge',           # hinge loss for linear SVM\n",
    "    penalty='l2',           # L2 regularization to prevent overfitting\n",
    "    alpha=1e-4,             # Learning rate (small value for fine-grained updates)\n",
    "    max_iter=1000,          # Number of passes over the training data\n",
    "    random_state=42,        # Seed for reproducible results\n",
    "    learning_rate='optimal' # Automatically adjusts the learning rate based on the training data\n",
    ")\n",
    "\n",
    "#SVM\n",
    "start = time.time()\n",
    "clf.fit(X_train_01, y_train_01)\n",
    "end = time.time()\n",
    "clf.score(X_train_01, y_train_01)\n",
    "time_taken = end - start\n",
    "with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "joblib.dump(clf, 'svm_level_01.joblib')\n",
    "\n",
    "\n",
    "clf = loaded_model = joblib.load('svm_level_01.joblib')\n",
    "\n",
    "\n",
    "#SVM\n",
    "start = time.time()\n",
    "preds_svm_01 = clf.predict(X_test_01)\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "print('---------------------------------------------------------------------------------')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0.0     1.0  2.0    3.0  4.0\n",
      "0.0  11056.0   261.0  0.0  111.0  0.0\n",
      "1.0     57.0  8048.0  0.0    6.0  0.0\n",
      "2.0     30.0  1950.0  1.0  170.0  0.0\n",
      "3.0     63.0    28.0  1.0  482.0  0.0\n",
      "4.0      8.0     2.0  0.0    4.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.8792081874495018\n",
      "Precision total:  0.5783299354191134\n",
      "Recall total:  0.5599734594264867\n",
      "F1 total:  0.5135936316789838\n",
      "BACC total:  0.5599734594264867\n",
      "MCC total:  0.8036574573652664\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    }
   ],
   "source": [
    "with open(output_file_name, \"a\") as f: print('-------------------------------------------------------', file = f)\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('CONFUSION MATRIX')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "with open(output_file_name, \"a\") as f: print('SVM', file = f)\n",
    "pred_label = preds_svm_01\n",
    "\n",
    "# pred_label = ypred\n",
    "#pred_label = label[ypred]\n",
    "\n",
    "confusion_matrix = pd.crosstab(y_test_01, pred_label,rownames=['Actual ALERT'],colnames = ['Predicted ALERT'], dropna=False).sort_index(axis=0).sort_index(axis=1)\n",
    "all_unique_values = sorted(set(pred_label) | set(y_test_01))\n",
    "z = np.zeros((len(all_unique_values), len(all_unique_values)))\n",
    "rows, cols = confusion_matrix.shape\n",
    "z[:rows, :cols] = confusion_matrix\n",
    "confusion_matrix  = pd.DataFrame(z, columns=all_unique_values, index=all_unique_values)\n",
    "# confusion_matrix.to_csv('Ensemble_conf_matrix.csv')\n",
    "# with open(output_file_name, \"a\") as f:print(confusion_matrix,file=f)\n",
    "print(confusion_matrix)\n",
    "with open(output_file_name, \"a\") as f: print('Confusion Matrix', file = f)\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print(confusion_matrix, file = f)\n",
    "\n",
    "\n",
    "FP = confusion_matrix.sum(axis=0) - np.diag(confusion_matrix)\n",
    "FN = confusion_matrix.sum(axis=1) - np.diag(confusion_matrix)\n",
    "TP = np.diag(confusion_matrix)\n",
    "TN = confusion_matrix.values.sum() - (FP + FN + TP)\n",
    "TP_total = sum(TP)\n",
    "TN_total = sum(TN)\n",
    "FP_total = sum(FP)\n",
    "FN_total = sum(FN)\n",
    "\n",
    "TP_total = np.array(TP_total,dtype=np.float64)\n",
    "TN_total = np.array(TN_total,dtype=np.float64)\n",
    "FP_total = np.array(FP_total,dtype=np.float64)\n",
    "FN_total = np.array(FN_total,dtype=np.float64)\n",
    "\n",
    "\n",
    "\n",
    "#----------------------------------------------------------------#----------------------------------------------------------------\n",
    "\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('METRICS')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "\n",
    "# Acc = ACC(TP_total,TN_total, FP_total, FN_total)\n",
    "# Precision = PRECISION(TP_total, FP_total)\n",
    "# Recall = RECALL(TP_total, FN_total)\n",
    "# F1 = F1(Recall,Precision)\n",
    "# BACC = BACC(TP_total,TN_total, FP_total, FN_total)\n",
    "# MCC = MCC(TP_total,TN_total, FP_total, FN_total)\n",
    "\n",
    "\n",
    "Acc = accuracy_score(y_test_01, pred_label)\n",
    "Precision = precision_score(y_test_01, pred_label, average='macro')\n",
    "Recall = recall_score(y_test_01, pred_label, average='macro')\n",
    "F1 =  f1_score(y_test_01, pred_label, average='macro')\n",
    "BACC = balanced_accuracy_score(y_test_01, pred_label)\n",
    "MCC = matthews_corrcoef(y_test_01, pred_label)\n",
    "\n",
    "\n",
    "\n",
    "svm_acc_01 = Acc\n",
    "svm_pre_01 = Precision\n",
    "svm_rec_01 = Recall\n",
    "svm_f1_01 = F1\n",
    "svm_bacc_01 = BACC\n",
    "svm_mcc_01 = MCC\n",
    "\n",
    "# with open(output_file_name, \"a\") as f:print('Accuracy total: ', Acc,file=f)\n",
    "print('Accuracy total: ', Acc)\n",
    "print('Precision total: ', Precision )\n",
    "print('Recall total: ', Recall )\n",
    "print('F1 total: ', F1 )\n",
    "print('BACC total: ', BACC)\n",
    "print('MCC total: ', MCC)\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('Accuracy total: ', Acc, file = f)\n",
    "with open(output_file_name, \"a\") as f: print('Precision total: ', Precision, file = f)\n",
    "with open(output_file_name, \"a\") as f: print('Recall total: ', Recall , file = f)\n",
    "with open(output_file_name, \"a\") as f: print('F1 total: ', F1, file = f)\n",
    "with open(output_file_name, \"a\") as f: print('BACC total: ', BACC , file = f)\n",
    "with open(output_file_name, \"a\") as f: print('MCC total: ', MCC, file = f)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Defining RF Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Training RF\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Prediction RF\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0.0     1.0     2.0    3.0  4.0\n",
      "0.0  11210.0    70.0    54.0   94.0  0.0\n",
      "1.0     48.0  8050.0     8.0    5.0  0.0\n",
      "2.0     45.0     5.0  2097.0    4.0  0.0\n",
      "3.0     64.0     1.0     8.0  501.0  0.0\n",
      "4.0     10.0     0.0     0.0    4.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9811473202262322\n",
      "Precision total:  0.7535358004758851\n",
      "Recall total:  0.7642242184751191\n",
      "F1 total:  0.7587357911044557\n",
      "BACC total:  0.7642242184751191\n",
      "MCC total:  0.9683303737241699\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining RF Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "#Random Forest\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.multioutput import MultiOutputClassifier\n",
    "rf = RandomForestClassifier(max_depth = 5,  n_estimators = 10, min_samples_split = 2, n_jobs = -1)\n",
    "#------------------------------------------------------------------------------\n",
    "\n",
    "if True == True:\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training RF')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Training RF', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    #RF\n",
    "    start = time.time()\n",
    "    model_rf_01 = rf.fit(X_train_01,y_train_01)\n",
    "    end = time.time()\n",
    "\n",
    "    # # Create the StratifiedKFold object\n",
    "    # stratified_kfold = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n",
    "    # # Perform cross-validation\n",
    "    # cv_scores = cross_val_score(model_rf_01, X_train_01, y_train, cv=stratified_kfold, scoring='accuracy')\n",
    "    # # Print the cross-validation scores\n",
    "    # print(\"Cross-validation scores:\", cv_scores)\n",
    "    # print(\"Mean accuracy:\", cv_scores.mean())\n",
    "    # with open(output_file_name, \"a\") as f: print('mean accuracy', cv_scores.mean() , file = f)\n",
    "\n",
    "\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "    joblib.dump(model_rf_01, 'rf_base_model_01.joblib')\n",
    "\n",
    "if 1 == 1:\n",
    "    model_rf_01  = joblib.load('rf_base_model_01.joblib')\n",
    "\n",
    "if 1 == 1:\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Prediction RF')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Prediction RF', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    #RF\n",
    "    start = time.time()\n",
    "    preds_rf_01 = model_rf_01.predict(X_test_01)\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('-------------------------------------------------------', file = f)\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('CONFUSION MATRIX')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "with open(output_file_name, \"a\") as f: print('RF', file = f)\n",
    "pred_label = preds_rf_01\n",
    "\n",
    "# pred_label = ypred\n",
    "#pred_label = label[ypred]\n",
    "\n",
    "confusion_matrix = pd.crosstab(y_test_01, pred_label,rownames=['Actual ALERT'],colnames = ['Predicted ALERT'], dropna=False).sort_index(axis=0).sort_index(axis=1)\n",
    "all_unique_values = sorted(set(pred_label) | set(y_test_01))\n",
    "z = np.zeros((len(all_unique_values), len(all_unique_values)))\n",
    "rows, cols = confusion_matrix.shape\n",
    "z[:rows, :cols] = confusion_matrix\n",
    "confusion_matrix  = pd.DataFrame(z, columns=all_unique_values, index=all_unique_values)\n",
    "# confusion_matrix.to_csv('Ensemble_conf_matrix.csv')\n",
    "# with open(output_file_name, \"a\") as f:print(confusion_matrix,file=f)\n",
    "print(confusion_matrix)\n",
    "with open(output_file_name, \"a\") as f: print('Confusion Matrix', file = f)\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print(confusion_matrix, file = f)\n",
    "\n",
    "\n",
    "FP = confusion_matrix.sum(axis=0) - np.diag(confusion_matrix)\n",
    "FN = confusion_matrix.sum(axis=1) - np.diag(confusion_matrix)\n",
    "TP = np.diag(confusion_matrix)\n",
    "TN = confusion_matrix.values.sum() - (FP + FN + TP)\n",
    "TP_total = sum(TP)\n",
    "TN_total = sum(TN)\n",
    "FP_total = sum(FP)\n",
    "FN_total = sum(FN)\n",
    "\n",
    "TP_total = np.array(TP_total,dtype=np.float64)\n",
    "TN_total = np.array(TN_total,dtype=np.float64)\n",
    "FP_total = np.array(FP_total,dtype=np.float64)\n",
    "FN_total = np.array(FN_total,dtype=np.float64)\n",
    "\n",
    "\n",
    "\n",
    "#----------------------------------------------------------------#----------------------------------------------------------------\n",
    "\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('METRICS')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "\n",
    "# Acc = ACC(TP_total,TN_total, FP_total, FN_total)\n",
    "# Precision = PRECISION(TP_total, FP_total)\n",
    "# Recall = RECALL(TP_total, FN_total)\n",
    "# F1 = F1(Recall,Precision)\n",
    "# BACC = BACC(TP_total,TN_total, FP_total, FN_total)\n",
    "# MCC = MCC(TP_total,TN_total, FP_total, FN_total)\n",
    "\n",
    "\n",
    "Acc = accuracy_score(y_test_01, pred_label)\n",
    "Precision = precision_score(y_test_01, pred_label, average='macro')\n",
    "Recall = recall_score(y_test_01, pred_label, average='macro')\n",
    "F1 =  f1_score(y_test_01, pred_label, average='macro')\n",
    "BACC = balanced_accuracy_score(y_test_01, pred_label)\n",
    "MCC = matthews_corrcoef(y_test_01, pred_label)\n",
    "\n",
    "rf_acc_01 = Acc\n",
    "rf_pre_01 = Precision\n",
    "rf_rec_01 = Recall\n",
    "rf_f1_01 = F1\n",
    "rf_bacc_01 = BACC\n",
    "rf_mcc_01 = MCC\n",
    "\n",
    "\n",
    "# with open(output_file_name, \"a\") as f:print('Accuracy total: ', Acc,file=f)\n",
    "print('Accuracy total: ', Acc)\n",
    "print('Precision total: ', Precision )\n",
    "print('Recall total: ', Recall )\n",
    "print('F1 total: ', F1 )\n",
    "print('BACC total: ', BACC)\n",
    "print('MCC total: ', MCC)\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('Accuracy total: ', Acc, file = f)\n",
    "with open(output_file_name, \"a\") as f: print('Precision total: ', Precision, file = f)\n",
    "with open(output_file_name, \"a\") as f: print('Recall total: ', Recall , file = f)\n",
    "with open(output_file_name, \"a\") as f: print('F1 total: ', F1, file = f)\n",
    "with open(output_file_name, \"a\") as f: print('BACC total: ', BACC , file = f)\n",
    "with open(output_file_name, \"a\") as f: print('MCC total: ', MCC, file = f)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9811473202262322"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf_acc_01"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Defining LGBM Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Training LGBM\n",
      "---------------------------------------------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prediction LGBM\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0.0     1.0     2.0    3.0  4.0\n",
      "0.0  11213.0    70.0    54.0   91.0  0.0\n",
      "1.0     50.0  8053.0     4.0    4.0  0.0\n",
      "2.0     45.0     6.0  2096.0    4.0  0.0\n",
      "3.0     66.0     1.0     8.0  499.0  0.0\n",
      "4.0     10.0     0.0     0.0    4.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9812819822246163\n",
      "Precision total:  0.7547768376013453\n",
      "Recall total:  0.7635608505955311\n",
      "F1 total:  0.7590703045266477\n",
      "BACC total:  0.7635608505955311\n",
      "MCC total:  0.9685440983622206\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    }
   ],
   "source": [
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining LGBM Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "#LGBM\n",
    "from lightgbm import LGBMClassifier\n",
    "lgbm = LGBMClassifier()\n",
    "\n",
    "\n",
    "\n",
    "if 1 == 1 and 0 == 0:\n",
    "\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training LGBM')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Training LGBM', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    start = time.time()\n",
    "    lgbm.fit(X_train_01, y_train_01)\n",
    "    end = time.time()\n",
    "\n",
    "    # # Create the StratifiedKFold object\n",
    "    # stratified_kfold = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n",
    "    # # Perform cross-validation\n",
    "    # cv_scores = cross_val_score(lgbm, X_train, y_train, cv=stratified_kfold, scoring='accuracy')\n",
    "    # # Print the cross-validation scores\n",
    "    # print(\"Cross-validation scores:\", cv_scores)\n",
    "    # print(\"Mean accuracy:\", cv_scores.mean())\n",
    "    # with open(output_file_name, \"a\") as f: print('mean accuracy', cv_scores.mean() , file = f)\n",
    "\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "    joblib.dump(lgbm, 'lgbm_01.joblib')\n",
    "\n",
    "if 1 == 1:\n",
    "    lgbm = joblib.load('lgbm_01.joblib')\n",
    "\n",
    "\n",
    "if 1 == 1:\n",
    "\n",
    "    print('Prediction LGBM')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Prediction LGBM', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    #LGBM\n",
    "    start = time.time()\n",
    "    preds_lgbm_01 = lgbm.predict(X_test_01)\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('CONFUSION MATRIX')\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    with open(output_file_name, \"a\") as f: print('LGBM', file = f)\n",
    "    pred_label = preds_lgbm_01\n",
    "\n",
    "    # pred_label = ypred\n",
    "    #pred_label = label[ypred]\n",
    "\n",
    "    confusion_matrix = pd.crosstab(y_test_01, pred_label,rownames=['Actual ALERT'],colnames = ['Predicted ALERT'], dropna=False).sort_index(axis=0).sort_index(axis=1)\n",
    "    all_unique_values = sorted(set(pred_label) | set(y_test_01))\n",
    "    z = np.zeros((len(all_unique_values), len(all_unique_values)))\n",
    "    rows, cols = confusion_matrix.shape\n",
    "    z[:rows, :cols] = confusion_matrix\n",
    "    confusion_matrix  = pd.DataFrame(z, columns=all_unique_values, index=all_unique_values)\n",
    "    # confusion_matrix.to_csv('Ensemble_conf_matrix.csv')\n",
    "    # with open(output_file_name, \"a\") as f:print(confusion_matrix,file=f)\n",
    "    print(confusion_matrix)\n",
    "    with open(output_file_name, \"a\") as f: print('Confusion Matrix', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print(confusion_matrix, file = f)\n",
    "\n",
    "\n",
    "    FP = confusion_matrix.sum(axis=0) - np.diag(confusion_matrix)\n",
    "    FN = confusion_matrix.sum(axis=1) - np.diag(confusion_matrix)\n",
    "    TP = np.diag(confusion_matrix)\n",
    "    TN = confusion_matrix.values.sum() - (FP + FN + TP)\n",
    "    TP_total = sum(TP)\n",
    "    TN_total = sum(TN)\n",
    "    FP_total = sum(FP)\n",
    "    FN_total = sum(FN)\n",
    "\n",
    "    TP_total = np.array(TP_total,dtype=np.float64)\n",
    "    TN_total = np.array(TN_total,dtype=np.float64)\n",
    "    FP_total = np.array(FP_total,dtype=np.float64)\n",
    "    FN_total = np.array(FN_total,dtype=np.float64)\n",
    "\n",
    "\n",
    "\n",
    "    #----------------------------------------------------------------#----------------------------------------------------------------\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('METRICS')\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "    # Acc = ACC(TP_total,TN_total, FP_total, FN_total)\n",
    "    # Precision = PRECISION(TP_total, FP_total)\n",
    "    # Recall = RECALL(TP_total, FN_total)\n",
    "    # F1 = F1(Recall,Precision)\n",
    "    # BACC = BACC(TP_total,TN_total, FP_total, FN_total)\n",
    "    # MCC = MCC(TP_total,TN_total, FP_total, FN_total)\n",
    "\n",
    "\n",
    "    Acc = accuracy_score(y_test_01, pred_label)\n",
    "    lgbm_acc_01 = Acc\n",
    "    Precision = precision_score(y_test_01, pred_label, average='macro')\n",
    "    Recall = recall_score(y_test_01, pred_label, average='macro')\n",
    "    F1 =  f1_score(y_test_01, pred_label, average='macro')\n",
    "    BACC = balanced_accuracy_score(y_test_01, pred_label)\n",
    "    MCC = matthews_corrcoef(y_test_01, pred_label)\n",
    "\n",
    "    lgbm_acc_01 = Acc\n",
    "    lgbm_pre_01 = Precision\n",
    "    lgbm_rec_01 = Recall\n",
    "    lgbm_f1_01 = F1\n",
    "    lgbm_bacc_01 = BACC\n",
    "    lgbm_mcc_01 = MCC\n",
    "\n",
    "    # with open(output_file_name, \"a\") as f:print('Accuracy total: ', Acc,file=f)\n",
    "    print('Accuracy total: ', Acc)\n",
    "    print('Precision total: ', Precision )\n",
    "    print('Recall total: ', Recall )\n",
    "    print('F1 total: ', F1 )\n",
    "    print('BACC total: ', BACC)\n",
    "    print('MCC total: ', MCC)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Accuracy total: ', Acc, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Precision total: ', Precision, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Recall total: ', Recall , file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('F1 total: ', F1, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('BACC total: ', BACC , file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('MCC total: ', MCC, file = f)\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "# lgbm_acc_01"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Defining MLP Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Training MLP\n",
      "---------------------------------------------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0.0     1.0     2.0    3.0  4.0\n",
      "0.0  11210.0    70.0    55.0   93.0  0.0\n",
      "1.0     49.0  8053.0     4.0    5.0  0.0\n",
      "2.0     46.0     5.0  2097.0    3.0  0.0\n",
      "3.0     66.0     1.0     9.0  498.0  0.0\n",
      "4.0     10.0     0.0     0.0    4.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9811473202262322\n",
      "Precision total:  0.7540175171198078\n",
      "Recall total:  0.7632528959239488\n",
      "F1 total:  0.7585292769243145\n",
      "BACC total:  0.7632528959239488\n",
      "MCC total:  0.9683222763644737\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "#MLP\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining MLP Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.multioutput import MultiOutputClassifier\n",
    "import time\n",
    "\n",
    "# create MLPClassifier instance\n",
    "mlp = MLPClassifier(hidden_layer_sizes=(100,), max_iter=200, random_state=1)\n",
    "\n",
    "if 1 == 1 and 0 == 0:\n",
    "\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training MLP')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Training MLP', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "    start = time.time()\n",
    "    MLP = mlp.fit(X_train_01, y_train_01)\n",
    "    end = time.time()\n",
    "\n",
    "    # # Create the StratifiedKFold object\n",
    "    # stratified_kfold = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n",
    "    # # Perform cross-validation\n",
    "    # cv_scores = cross_val_score(MLP, X_train, y_train, cv=stratified_kfold, scoring='accuracy')\n",
    "    # # Print the cross-validation scores\n",
    "    # print(\"Cross-validation scores:\", cv_scores)\n",
    "    # print(\"Mean accuracy:\", cv_scores.mean())\n",
    "    # with open(output_file_name, \"a\") as f: print('mean accuracy', cv_scores.mean() , file = f)\n",
    "\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "    joblib.dump(MLP, 'mlp_01.joblib')\n",
    "\n",
    "if 1 == 1:\n",
    "    MLP = joblib.load('mlp_01.joblib')\n",
    "\n",
    "\n",
    "if 1 == 1:\n",
    "\n",
    "    #MLP\n",
    "    start = time.time()\n",
    "    y_pred = MLP.predict_proba(X_test_01)\n",
    "    preds_mlp_01 = np.argmax(y_pred,axis = 1)\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "#MLP\n",
    "if 1 == 1:\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('MLP 01 model', file = f)\n",
    "\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('CONFUSION MATRIX')\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "    pred_label = preds_mlp_01\n",
    "    # pred_label = label[ypred]\n",
    "\n",
    "    confusion_matrix = pd.crosstab(y_test_01, pred_label,rownames=['Actual ALERT'],colnames = ['Predicted ALERT'], dropna=False).sort_index(axis=0).sort_index(axis=1)\n",
    "    all_unique_values = sorted(set(pred_label) | set(y_test_01))\n",
    "    z = np.zeros((len(all_unique_values), len(all_unique_values)))\n",
    "    rows, cols = confusion_matrix.shape\n",
    "    z[:rows, :cols] = confusion_matrix\n",
    "    confusion_matrix  = pd.DataFrame(z, columns=all_unique_values, index=all_unique_values)\n",
    "    # confusion_matrix.to_csv('Ensemble_conf_matrix.csv')\n",
    "    # with open(output_file_name, \"a\") as f:print(confusion_matrix,file=f)\n",
    "    print(confusion_matrix)\n",
    "    with open(output_file_name, \"a\") as f: print('Confusion Matrix', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print(confusion_matrix, file = f)\n",
    "\n",
    "\n",
    "    FP = confusion_matrix.sum(axis=0) - np.diag(confusion_matrix)\n",
    "    FN = confusion_matrix.sum(axis=1) - np.diag(confusion_matrix)\n",
    "    TP = np.diag(confusion_matrix)\n",
    "    TN = confusion_matrix.values.sum() - (FP + FN + TP)\n",
    "    TP_total = sum(TP)\n",
    "    TN_total = sum(TN)\n",
    "    FP_total = sum(FP)\n",
    "    FN_total = sum(FN)\n",
    "\n",
    "    TP_total = np.array(TP_total,dtype=np.float64)\n",
    "    TN_total = np.array(TN_total,dtype=np.float64)\n",
    "    FP_total = np.array(FP_total,dtype=np.float64)\n",
    "    FN_total = np.array(FN_total,dtype=np.float64)\n",
    "\n",
    "\n",
    "\n",
    "    #----------------------------------------------------------------#----------------------------------------------------------------\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('METRICS')\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "    # Acc = ACC(TP_total,TN_total, FP_total, FN_total)\n",
    "    # Precision = PRECISION(TP_total, FP_total)\n",
    "    # Recall = RECALL(TP_total, FN_total)\n",
    "    # F1 = F1(Recall,Precision)\n",
    "    # BACC = BACC(TP_total,TN_total, FP_total, FN_total)\n",
    "    # MCC = MCC(TP_total,TN_total, FP_total, FN_total)\n",
    "\n",
    "\n",
    "    Acc = accuracy_score(y_test_01, pred_label)\n",
    "    Precision = precision_score(y_test_01, pred_label, average='macro')\n",
    "    Recall = recall_score(y_test_01, pred_label, average='macro')\n",
    "    F1 =  f1_score(y_test_01, pred_label, average='macro')\n",
    "    BACC = balanced_accuracy_score(y_test_01, pred_label)\n",
    "    MCC = matthews_corrcoef(y_test_01, pred_label)\n",
    "\n",
    "\n",
    "    mlp_acc_01 = Acc\n",
    "    mlp_pre_01 = Precision\n",
    "    mlp_rec_01 = Recall\n",
    "    mlp_f1_01 = F1\n",
    "    mlp_bacc_01 = BACC\n",
    "    mlp_mcc_01 = MCC\n",
    "\n",
    "    # with open(output_file_name, \"a\") as f:print('Accuracy total: ', Acc,file=f)\n",
    "    print('Accuracy total: ', Acc)\n",
    "    print('Precision total: ', Precision )\n",
    "    print('Recall total: ', Recall )\n",
    "    print('F1 total: ', F1 )\n",
    "    print('BACC total: ', BACC)\n",
    "    print('MCC total: ', MCC)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Accuracy total: ', Acc, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Precision total: ', Precision, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Recall total: ', Recall , file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('F1 total: ', F1, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('BACC total: ', BACC , file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('MCC total: ', MCC, file = f)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "# mlp_acc_01"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Defining ADA Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Training ADA\n",
      "---------------------------------------------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prediction ADA\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0.0     1.0     2.0    3.0  4.0\n",
      "0.0  11246.0    70.0    43.0   69.0  0.0\n",
      "1.0     51.0  8017.0    39.0    4.0  0.0\n",
      "2.0     78.0   189.0  1880.0    4.0  0.0\n",
      "3.0    171.0     1.0     7.0  395.0  0.0\n",
      "4.0     10.0     0.0     0.0    4.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9667833737319328\n",
      "Precision total:  0.7452786169091856\n",
      "Recall total:  0.7069300802727743\n",
      "F1 total:  0.724398988880189\n",
      "BACC total:  0.7069300802727743\n",
      "MCC total:  0.943830575668782\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    }
   ],
   "source": [
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining ADA Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "#ADA\n",
    "from sklearn.multioutput import MultiOutputClassifier\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "import time\n",
    "abc = AdaBoostClassifier(n_estimators=50, learning_rate=1.0)\n",
    "\n",
    "if 1 == 1 and 0 == 0:\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training ADA')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Training ADA', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    #ADA\n",
    "\n",
    "\n",
    "    start = time.time()\n",
    "    ada = abc.fit(X_train_01, y_train_01)\n",
    "    end = time.time()\n",
    "\n",
    "    # # Create the StratifiedKFold object\n",
    "    # stratified_kfold = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n",
    "    # # Perform cross-validation\n",
    "    # cv_scores = cross_val_score(ada, X_train, y_train, cv=stratified_kfold, scoring='accuracy')\n",
    "    # # Print the cross-validation scores\n",
    "    # print(\"Cross-validation scores:\", cv_scores)\n",
    "    # print(\"Mean accuracy:\", cv_scores.mean())\n",
    "    # with open(output_file_name, \"a\") as f: print('mean accuracy', cv_scores.mean() , file = f)\n",
    "\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "\n",
    "    # Assuming 'model' is your trained model\n",
    "    joblib.dump(ada, 'ada_01.joblib')\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "if 1 == 1:\n",
    "    ada = joblib.load('ada_01.joblib')\n",
    "\n",
    "\n",
    "if 1 == 1:\n",
    "\n",
    "    print('Prediction ADA')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Prediction ADA', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    #ADA\n",
    "    start = time.time()\n",
    "    preds_ada_01 = ada.predict(X_test_01)\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "if 1 == 1:\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('ADA 01 model', file = f)\n",
    "\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('CONFUSION MATRIX')\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "    pred_label = preds_ada_01\n",
    "    # pred_label = label[ypred]\n",
    "\n",
    "    confusion_matrix = pd.crosstab(y_test_01, pred_label,rownames=['Actual ALERT'],colnames = ['Predicted ALERT'], dropna=False).sort_index(axis=0).sort_index(axis=1)\n",
    "    all_unique_values = sorted(set(pred_label) | set(y_test_01))\n",
    "    z = np.zeros((len(all_unique_values), len(all_unique_values)))\n",
    "    rows, cols = confusion_matrix.shape\n",
    "    z[:rows, :cols] = confusion_matrix\n",
    "    confusion_matrix  = pd.DataFrame(z, columns=all_unique_values, index=all_unique_values)\n",
    "    # confusion_matrix.to_csv('Ensemble_conf_matrix.csv')\n",
    "    # with open(output_file_name, \"a\") as f:print(confusion_matrix,file=f)\n",
    "    print(confusion_matrix)\n",
    "    with open(output_file_name, \"a\") as f: print('Confusion Matrix', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print(confusion_matrix, file = f)\n",
    "\n",
    "\n",
    "    FP = confusion_matrix.sum(axis=0) - np.diag(confusion_matrix)\n",
    "    FN = confusion_matrix.sum(axis=1) - np.diag(confusion_matrix)\n",
    "    TP = np.diag(confusion_matrix)\n",
    "    TN = confusion_matrix.values.sum() - (FP + FN + TP)\n",
    "    TP_total = sum(TP)\n",
    "    TN_total = sum(TN)\n",
    "    FP_total = sum(FP)\n",
    "    FN_total = sum(FN)\n",
    "\n",
    "    TP_total = np.array(TP_total,dtype=np.float64)\n",
    "    TN_total = np.array(TN_total,dtype=np.float64)\n",
    "    FP_total = np.array(FP_total,dtype=np.float64)\n",
    "    FN_total = np.array(FN_total,dtype=np.float64)\n",
    "\n",
    "\n",
    "\n",
    "    #----------------------------------------------------------------#----------------------------------------------------------------\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('METRICS')\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "    # Acc = ACC(TP_total,TN_total, FP_total, FN_total)\n",
    "    # Precision = PRECISION(TP_total, FP_total)\n",
    "    # Recall = RECALL(TP_total, FN_total)\n",
    "    # F1 = F1(Recall,Precision)\n",
    "    # BACC = BACC(TP_total,TN_total, FP_total, FN_total)\n",
    "    # MCC = MCC(TP_total,TN_total, FP_total, FN_total)\n",
    "\n",
    "\n",
    "    Acc = accuracy_score(y_test_01, pred_label)\n",
    "    Precision = precision_score(y_test_01, pred_label, average='macro')\n",
    "    Recall = recall_score(y_test_01, pred_label, average='macro')\n",
    "    F1 =  f1_score(y_test_01, pred_label, average='macro')\n",
    "    BACC = balanced_accuracy_score(y_test_01, pred_label)\n",
    "    MCC = matthews_corrcoef(y_test_01, pred_label)\n",
    "\n",
    "    ada_acc_01 = Acc\n",
    "    ada_pre_01 = Precision\n",
    "    ada_rec_01 = Recall\n",
    "    ada_f1_01 = F1\n",
    "    ada_bacc_01 = BACC\n",
    "    ada_mcc_01 = MCC\n",
    "\n",
    "    # with open(output_file_name, \"a\") as f:print('Accuracy total: ', Acc,file=f)\n",
    "    print('Accuracy total: ', Acc)\n",
    "    print('Precision total: ', Precision )\n",
    "    print('Recall total: ', Recall )\n",
    "    print('F1 total: ', F1 )\n",
    "    print('BACC total: ', BACC)\n",
    "    print('MCC total: ', MCC)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Accuracy total: ', Acc, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Precision total: ', Precision, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Recall total: ', Recall , file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('F1 total: ', F1, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('BACC total: ', BACC , file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('MCC total: ', MCC, file = f)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Defining KNN Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Training KNN\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0.0     1.0     2.0    3.0  4.0\n",
      "0.0  11225.0    71.0    44.0   88.0  0.0\n",
      "1.0     55.0  8048.0     4.0    4.0  0.0\n",
      "2.0     77.0     6.0  2065.0    3.0  0.0\n",
      "3.0     70.0     1.0    11.0  492.0  0.0\n",
      "4.0     10.0     0.0     0.0    4.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.979890474907981\n",
      "Precision total:  0.7553148767605853\n",
      "Recall total:  0.7583261670574462\n",
      "F1 total:  0.7567766366742548\n",
      "BACC total:  0.7583261670574462\n",
      "MCC total:  0.9661461308387623\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    }
   ],
   "source": [
    "#KNN\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining KNN Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "knn_clf_01=KNeighborsClassifier(n_neighbors = 5)\n",
    "\n",
    "if 1 == 1 and 0 == 0:\n",
    "\n",
    "    #KNN\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training KNN')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Training KNN', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    start = time.time()\n",
    "    knn_clf_01.fit(X_train_01,y_train_01)\n",
    "    end = time.time()\n",
    "\n",
    "\n",
    "    # # Create the StratifiedKFold object\n",
    "    # stratified_kfold = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n",
    "    # # Perform cross-validation\n",
    "    # cv_scores = cross_val_score(knn_clf, X_train, y_train, cv=stratified_kfold, scoring='accuracy')\n",
    "    # # Print the cross-validation scores\n",
    "    # print(\"Cross-validation scores:\", cv_scores)\n",
    "    # print(\"Mean accuracy:\", cv_scores.mean())\n",
    "    # with open(output_file_name, \"a\") as f: print('mean accuracy', cv_scores.mean() , file = f)\n",
    "\n",
    "\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "    joblib.dump(knn_clf_01, 'knn_01.joblib')\n",
    "\n",
    "\n",
    "if load_model_knn == 1:\n",
    "    knn_clf_01 = joblib.load('knn_01.joblib')\n",
    "\n",
    "if use_model_knn == 1:\n",
    "\n",
    "    #KNN\n",
    "    start = time.time()\n",
    "    preds_knn =knn_clf_01.predict(X_test_01)\n",
    "    preds_knn\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "#MLP\n",
    "if 1 == 1:\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('KNN 01 model', file = f)\n",
    "\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('CONFUSION MATRIX')\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "    pred_label = preds_knn\n",
    "    # pred_label = label[ypred]\n",
    "\n",
    "    confusion_matrix = pd.crosstab(y_test_01, pred_label,rownames=['Actual ALERT'],colnames = ['Predicted ALERT'], dropna=False).sort_index(axis=0).sort_index(axis=1)\n",
    "    all_unique_values = sorted(set(pred_label) | set(y_test_01))\n",
    "    z = np.zeros((len(all_unique_values), len(all_unique_values)))\n",
    "    rows, cols = confusion_matrix.shape\n",
    "    z[:rows, :cols] = confusion_matrix\n",
    "    confusion_matrix  = pd.DataFrame(z, columns=all_unique_values, index=all_unique_values)\n",
    "    # confusion_matrix.to_csv('Ensemble_conf_matrix.csv')\n",
    "    # with open(output_file_name, \"a\") as f:print(confusion_matrix,file=f)\n",
    "    print(confusion_matrix)\n",
    "    with open(output_file_name, \"a\") as f: print('Confusion Matrix', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print(confusion_matrix, file = f)\n",
    "\n",
    "\n",
    "    FP = confusion_matrix.sum(axis=0) - np.diag(confusion_matrix)\n",
    "    FN = confusion_matrix.sum(axis=1) - np.diag(confusion_matrix)\n",
    "    TP = np.diag(confusion_matrix)\n",
    "    TN = confusion_matrix.values.sum() - (FP + FN + TP)\n",
    "    TP_total = sum(TP)\n",
    "    TN_total = sum(TN)\n",
    "    FP_total = sum(FP)\n",
    "    FN_total = sum(FN)\n",
    "\n",
    "    TP_total = np.array(TP_total,dtype=np.float64)\n",
    "    TN_total = np.array(TN_total,dtype=np.float64)\n",
    "    FP_total = np.array(FP_total,dtype=np.float64)\n",
    "    FN_total = np.array(FN_total,dtype=np.float64)\n",
    "\n",
    "\n",
    "\n",
    "    #----------------------------------------------------------------#----------------------------------------------------------------\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('METRICS')\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "    # Acc = ACC(TP_total,TN_total, FP_total, FN_total)\n",
    "    # Precision = PRECISION(TP_total, FP_total)\n",
    "    # Recall = RECALL(TP_total, FN_total)\n",
    "    # F1 = F1(Recall,Precision)\n",
    "    # BACC = BACC(TP_total,TN_total, FP_total, FN_total)\n",
    "    # MCC = MCC(TP_total,TN_total, FP_total, FN_total)\n",
    "\n",
    "\n",
    "    Acc = accuracy_score(y_test_01, pred_label)\n",
    "    Precision = precision_score(y_test_01, pred_label, average='macro')\n",
    "    Recall = recall_score(y_test_01, pred_label, average='macro')\n",
    "    F1 =  f1_score(y_test_01, pred_label, average='macro')\n",
    "    BACC = balanced_accuracy_score(y_test_01, pred_label)\n",
    "    MCC = matthews_corrcoef(y_test_01, pred_label)\n",
    "\n",
    "\n",
    "    knn_acc_01 = Acc\n",
    "    knn_pre_01 = Precision\n",
    "    knn_rec_01 = Recall\n",
    "    knn_f1_01 = F1\n",
    "    knn_bacc_01 = BACC\n",
    "    knn_mcc_01 = MCC    \n",
    "\n",
    "    # with open(output_file_name, \"a\") as f:print('Accuracy total: ', Acc,file=f)\n",
    "    print('Accuracy total: ', Acc)\n",
    "    print('Precision total: ', Precision )\n",
    "    print('Recall total: ', Recall )\n",
    "    print('F1 total: ', F1 )\n",
    "    print('BACC total: ', BACC)\n",
    "    print('MCC total: ', MCC)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Accuracy total: ', Acc, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Precision total: ', Precision, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Recall total: ', Recall , file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('F1 total: ', F1, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('BACC total: ', BACC , file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('MCC total: ', MCC, file = f)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Defining Logistic Regression Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Training LR \n",
      "---------------------------------------------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0.0     1.0     2.0    3.0  4.0\n",
      "0.0  11073.0   224.0    37.0   94.0  0.0\n",
      "1.0     46.0  8054.0     7.0    4.0  0.0\n",
      "2.0     30.0    96.0  1907.0  118.0  0.0\n",
      "3.0     62.0     4.0    27.0  481.0  0.0\n",
      "4.0      8.0     2.0     0.0    4.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9657509650776551\n",
      "Precision total:  0.7196703991990359\n",
      "Recall total:  0.737290387200625\n",
      "F1 total:  0.7265746441494881\n",
      "BACC total:  0.737290387200625\n",
      "MCC total:  0.9427310738311581\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "#Logistic Regression\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining Logistic Regression Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "logreg_01 = LogisticRegression()\n",
    "\n",
    "if 1 == 1 and 0 == 0:\n",
    "\n",
    "    #KNN\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training LR ')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Training LR', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    start = time.time()\n",
    "    logreg_01.fit(X_train_01,y_train_01)\n",
    "    end = time.time()\n",
    "\n",
    "\n",
    "    # # Create the StratifiedKFold object\n",
    "    # stratified_kfold = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n",
    "    # # Perform cross-validation\n",
    "    # cv_scores = cross_val_score(knn_clf, X_train, y_train, cv=stratified_kfold, scoring='accuracy')\n",
    "    # # Print the cross-validation scores\n",
    "    # print(\"Cross-validation scores:\", cv_scores)\n",
    "    # print(\"Mean accuracy:\", cv_scores.mean())\n",
    "    # with open(output_file_name, \"a\") as f: print('mean accuracy', cv_scores.mean() , file = f)\n",
    "\n",
    "\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "    joblib.dump(logreg_01, 'logreg_01.joblib')\n",
    "\n",
    "\n",
    "if 1 == 1:\n",
    "    logreg_01 = joblib.load('logreg_01.joblib')\n",
    "\n",
    "if 1 == 1:\n",
    "\n",
    "    #lR\n",
    "    start = time.time()\n",
    "    preds_logreg =logreg_01.predict(X_test_01)\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "#LR\n",
    "if 1 == 1:\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('LR 01 model', file = f)\n",
    "\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('CONFUSION MATRIX')\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "    pred_label = preds_logreg\n",
    "    # pred_label = label[ypred]\n",
    "\n",
    "    confusion_matrix = pd.crosstab(y_test_01, pred_label,rownames=['Actual ALERT'],colnames = ['Predicted ALERT'], dropna=False).sort_index(axis=0).sort_index(axis=1)\n",
    "    all_unique_values = sorted(set(pred_label) | set(y_test_01))\n",
    "    z = np.zeros((len(all_unique_values), len(all_unique_values)))\n",
    "    rows, cols = confusion_matrix.shape\n",
    "    z[:rows, :cols] = confusion_matrix\n",
    "    confusion_matrix  = pd.DataFrame(z, columns=all_unique_values, index=all_unique_values)\n",
    "    # confusion_matrix.to_csv('Ensemble_conf_matrix.csv')\n",
    "    # with open(output_file_name, \"a\") as f:print(confusion_matrix,file=f)\n",
    "    print(confusion_matrix)\n",
    "    with open(output_file_name, \"a\") as f: print('Confusion Matrix', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print(confusion_matrix, file = f)\n",
    "\n",
    "\n",
    "    FP = confusion_matrix.sum(axis=0) - np.diag(confusion_matrix)\n",
    "    FN = confusion_matrix.sum(axis=1) - np.diag(confusion_matrix)\n",
    "    TP = np.diag(confusion_matrix)\n",
    "    TN = confusion_matrix.values.sum() - (FP + FN + TP)\n",
    "    TP_total = sum(TP)\n",
    "    TN_total = sum(TN)\n",
    "    FP_total = sum(FP)\n",
    "    FN_total = sum(FN)\n",
    "\n",
    "    TP_total = np.array(TP_total,dtype=np.float64)\n",
    "    TN_total = np.array(TN_total,dtype=np.float64)\n",
    "    FP_total = np.array(FP_total,dtype=np.float64)\n",
    "    FN_total = np.array(FN_total,dtype=np.float64)\n",
    "\n",
    "\n",
    "\n",
    "    #----------------------------------------------------------------#----------------------------------------------------------------\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('METRICS')\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "    # Acc = ACC(TP_total,TN_total, FP_total, FN_total)\n",
    "    # Precision = PRECISION(TP_total, FP_total)\n",
    "    # Recall = RECALL(TP_total, FN_total)\n",
    "    # F1 = F1(Recall,Precision)\n",
    "    # BACC = BACC(TP_total,TN_total, FP_total, FN_total)\n",
    "    # MCC = MCC(TP_total,TN_total, FP_total, FN_total)\n",
    "\n",
    "\n",
    "    Acc = accuracy_score(y_test_01, pred_label)\n",
    "    Precision = precision_score(y_test_01, pred_label, average='macro')\n",
    "    Recall = recall_score(y_test_01, pred_label, average='macro')\n",
    "    F1 =  f1_score(y_test_01, pred_label, average='macro')\n",
    "    BACC = balanced_accuracy_score(y_test_01, pred_label)\n",
    "    MCC = matthews_corrcoef(y_test_01, pred_label)\n",
    "\n",
    "    lr_acc_01 = Acc\n",
    "    lr_pre_01 = Precision\n",
    "    lr_rec_01 = Recall\n",
    "    lr_f1_01 = F1\n",
    "    lr_bacc_01 = BACC\n",
    "    lr_mcc_01 = MCC\n",
    "\n",
    "    # with open(output_file_name, \"a\") as f:print('Accuracy total: ', Acc,file=f)\n",
    "    print('Accuracy total: ', Acc)\n",
    "    print('Precision total: ', Precision )\n",
    "    print('Recall total: ', Recall )\n",
    "    print('F1 total: ', F1 )\n",
    "    print('BACC total: ', BACC)\n",
    "    print('MCC total: ', MCC)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Accuracy total: ', Acc, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Precision total: ', Precision, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Recall total: ', Recall , file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('F1 total: ', F1, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('BACC total: ', BACC , file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('MCC total: ', MCC, file = f)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "# #Voting\n",
    "# from sklearn.ensemble import VotingClassifier\n",
    "# # model1 = LogisticRegression(random_state=1)\n",
    "# # model2 = tree.DecisionTreeClassifier(random_state=1)\n",
    "# voting = VotingClassifier(estimators=[\n",
    "#                                         ('ada', ada),\n",
    "#                                        ('rf', rf),\n",
    "#                                        ('svm', clf),\n",
    "#                                        ('knn', knn_clf), \n",
    "#                                        ('lgbm', lgbm),\n",
    "#                                       #  ('xgb', xgb_00),\n",
    "#                                        ('cat', cat_00),\n",
    "\n",
    "#                                          ('mlp', mlp)\n",
    "#                                         #  ,('dnn', dnn_01)\n",
    "\n",
    "#                                          ], voting='hard')\n",
    "# voting.fit(X_train_01,y_train_01)\n",
    "# # voring_acc = voting.score(X_test_01,y_test_01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "# preds_voting = voting(X_test_01,y_test_01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# print('---------------------------------------------------------------------------------')\n",
    "# print('CONFUSION MATRIX')\n",
    "# print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "# pred_label = preds_voting\n",
    "# # pred_label = label[ypred]\n",
    "\n",
    "# confusion_matrix = pd.crosstab(y_test_01, pred_label,rownames=['Actual ALERT'],colnames = ['Predicted ALERT'], dropna=False).sort_index(axis=0).sort_index(axis=1)\n",
    "# all_unique_values = sorted(set(pred_label) | set(y_test_01))\n",
    "# z = np.zeros((len(all_unique_values), len(all_unique_values)))\n",
    "# rows, cols = confusion_matrix.shape\n",
    "# z[:rows, :cols] = confusion_matrix\n",
    "# confusion_matrix  = pd.DataFrame(z, columns=all_unique_values, index=all_unique_values)\n",
    "# # confusion_matrix.to_csv('Ensemble_conf_matrix.csv')\n",
    "# # with open(output_file_name, \"a\") as f:print(confusion_matrix,file=f)\n",
    "# print(confusion_matrix)\n",
    "# with open(output_file_name, \"a\") as f: print('Confusion Matrix', file = f)\n",
    "\n",
    "# with open(output_file_name, \"a\") as f: print(confusion_matrix, file = f)\n",
    "\n",
    "\n",
    "# FP = confusion_matrix.sum(axis=0) - np.diag(confusion_matrix)\n",
    "# FN = confusion_matrix.sum(axis=1) - np.diag(confusion_matrix)\n",
    "# TP = np.diag(confusion_matrix)\n",
    "# TN = confusion_matrix.values.sum() - (FP + FN + TP)\n",
    "# TP_total = sum(TP)\n",
    "# TN_total = sum(TN)\n",
    "# FP_total = sum(FP)\n",
    "# FN_total = sum(FN)\n",
    "\n",
    "# TP_total = np.array(TP_total,dtype=np.float64)\n",
    "# TN_total = np.array(TN_total,dtype=np.float64)\n",
    "# FP_total = np.array(FP_total,dtype=np.float64)\n",
    "# FN_total = np.array(FN_total,dtype=np.float64)\n",
    "\n",
    "\n",
    "\n",
    "# #----------------------------------------------------------------#----------------------------------------------------------------\n",
    "\n",
    "# print('---------------------------------------------------------------------------------')\n",
    "# print('METRICS')\n",
    "# print('---------------------------------------------------------------------------------')\n",
    "\n",
    "# # Acc = ACC(TP_total,TN_total, FP_total, FN_total)\n",
    "# # Precision = PRECISION(TP_total, FP_total)\n",
    "# # Recall = RECALL(TP_total, FN_total)\n",
    "# # F1 = F1(Recall,Precision)\n",
    "# # BACC = BACC(TP_total,TN_total, FP_total, FN_total)\n",
    "# # MCC = MCC(TP_total,TN_total, FP_total, FN_total)\n",
    "\n",
    "\n",
    "# Acc = accuracy_score(y_test_01, pred_label)\n",
    "# Precision = precision_score(y_test_01, pred_label, average='macro')\n",
    "# Recall = recall_score(y_test_01, pred_label, average='macro')\n",
    "# F1 =  f1_score(y_test_01, pred_label, average='macro')\n",
    "# BACC = balanced_accuracy_score(y_test_01, pred_label)\n",
    "# MCC = matthews_corrcoef(y_test_01, pred_label)\n",
    "\n",
    "\n",
    "# voting_acc_01 = Acc\n",
    "# voting_pre_01 = Precision\n",
    "# voting_rec_01 = Recall\n",
    "# voting_f1_01 = F1\n",
    "# voting_bacc_01 = BACC\n",
    "# voting_mcc_01 = MCC\n",
    "\n",
    "# # with open(output_file_name, \"a\") as f:print('Accuracy total: ', Acc,file=f)\n",
    "# print('Accuracy total: ', Acc)\n",
    "# print('Precision total: ', Precision )\n",
    "# print('Recall total: ', Recall )\n",
    "# print('F1 total: ', F1 )\n",
    "# print('BACC total: ', BACC)\n",
    "# print('MCC total: ', MCC)\n",
    "\n",
    "# with open(output_file_name, \"a\") as f: print('Accuracy total: ', Acc, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Precision total: ', Precision, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Recall total: ', Recall , file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('F1 total: ', F1, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('BACC total: ', BACC , file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('MCC total: ', MCC, file = f)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from sklearn.calibration import CalibratedClassifierCV\n",
    "# with open(output_file_name, \"a\") as f: print('Generating Predictions', file = f)\n",
    "\n",
    "# if use_model_rf == 1:\n",
    "\n",
    "#     print('---------------------------------------------------------------------------------')\n",
    "#     print('Prediction RF')\n",
    "#     with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "#     with open(output_file_name, \"a\") as f: print('Prediction RF', file = f)\n",
    "#     print('---------------------------------------------------------------------------------')\n",
    "#     #RF\n",
    "#     start = time.time()\n",
    "#     preds_rf = rf.predict(X_test)\n",
    "#     preds_rf_prob = rf.predict_proba(X_test)\n",
    "#     end = time.time()\n",
    "#     time_taken = end - start\n",
    "#     with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "#     print('---------------------------------------------------------------------------------')\n",
    "\n",
    "# if use_model_svm == 1:\n",
    "\n",
    "#     print('Prediction SVM')\n",
    "#     with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "#     with open(output_file_name, \"a\") as f: print('Prediction SVM', file = f)\n",
    "#     print('---------------------------------------------------------------------------------')\n",
    "#     #SVM\n",
    "#     start = time.time()\n",
    "#     preds_svm = clf.predict(X_test)\n",
    "#     # preds_svm_prob = clf.predict_proba(X_test)\n",
    "\n",
    "#     #Since SVM does not deal with prob by nature we use a meta learner\n",
    "#     # https://stackoverflow.com/questions/55250963/how-to-get-probabilities-for-sgdclassifier-linearsvm\n",
    "\n",
    "#     model = CalibratedClassifierCV(clf)\n",
    "\n",
    "#     model.fit(X, y)\n",
    "#     preds_svm_prob = model.predict_proba(X)\n",
    "\n",
    "#     end = time.time()\n",
    "#     time_taken = end - start\n",
    "#     with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "#     print('---------------------------------------------------------------------------------')\n",
    "\n",
    "# if use_model_lgbm == 1:\n",
    "\n",
    "#     print('Prediction LGBM')\n",
    "#     with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "#     with open(output_file_name, \"a\") as f: print('Prediction LGBM', file = f)\n",
    "#     print('---------------------------------------------------------------------------------')\n",
    "#     #LGBM\n",
    "#     start = time.time()\n",
    "#     preds_lgbm = lgbm.predict(X_test)\n",
    "#     preds_lgbm_prob = lgbm.predict_proba(X_test)\n",
    "\n",
    "#     end = time.time()\n",
    "#     time_taken = end - start\n",
    "#     with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "#     print('---------------------------------------------------------------------------------')\n",
    "\n",
    "# if use_model_dnn == 1:\n",
    "\n",
    "#     print('Prediction DNN')\n",
    "#     with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "#     with open(output_file_name, \"a\") as f: print('Prediction DNN', file = f)\n",
    "#     print('---------------------------------------------------------------------------------')\n",
    "#     #DNN\n",
    "#     start = time.time()\n",
    "#     pred_dnn = dnn.predict(X_test)\n",
    "#     preds_dnn_prob = pred_dnn\n",
    "#     preds_dnn = np.argmax(pred_dnn,axis = 1)\n",
    "#     end = time.time()\n",
    "#     time_taken = end - start\n",
    "#     with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "#     print('---------------------------------------------------------------------------------')\n",
    "\n",
    "# if use_model_ada == 1:\n",
    "\n",
    "#     print('Prediction ADA')\n",
    "#     with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "#     with open(output_file_name, \"a\") as f: print('Prediction ADA', file = f)\n",
    "#     print('---------------------------------------------------------------------------------')\n",
    "#     #ADA\n",
    "#     start = time.time()\n",
    "#     preds_ada = ada.predict(X_test)\n",
    "#     preds_ada_prob = ada.predict_proba(X_test)\n",
    "\n",
    "#     end = time.time()\n",
    "#     time_taken = end - start\n",
    "#     with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "#     print('---------------------------------------------------------------------------------')\n",
    "#     print('Prediction MLP')\n",
    "#     with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "#     with open(output_file_name, \"a\") as f: print('Prediction MLP', file = f)\n",
    "#     print('---------------------------------------------------------------------------------')\n",
    "\n",
    "# if use_model_mlp == 1:\n",
    "\n",
    "#     #MLP\n",
    "#     start = time.time()\n",
    "#     y_pred = MLP.predict_proba(X_test)\n",
    "#     preds_mlp_prob = y_pred\n",
    "#     preds_mlp = np.argmax(y_pred,axis = 1)\n",
    "#     end = time.time()\n",
    "#     time_taken = end - start\n",
    "#     with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "#     print('---------------------------------------------------------------------------------')\n",
    "#     print('Prediction KNN')\n",
    "#     with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "#     with open(output_file_name, \"a\") as f: print('Prediction KNN', file = f)\n",
    "#     print('---------------------------------------------------------------------------------')\n",
    "\n",
    "# if use_model_knn == 1:\n",
    "\n",
    "#     #KNN\n",
    "#     start = time.time()\n",
    "#     preds_knn =knn_clf.predict(X_test)\n",
    "#     preds_knn_prob =knn_clf.predict_proba(X_test)\n",
    "\n",
    "#     preds_knn\n",
    "#     end = time.time()\n",
    "#     time_taken = end - start\n",
    "#     with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "#     with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:\tlearn: 1.2495200\ttest: 1.2489695\tbest: 1.2489695 (0)\ttotal: 59.5ms\tremaining: 5.89s\n",
      "10:\tlearn: 0.3443327\ttest: 0.3418232\tbest: 0.3418232 (10)\ttotal: 131ms\tremaining: 1.06s\n",
      "20:\tlearn: 0.1691488\ttest: 0.1659377\tbest: 0.1659377 (20)\ttotal: 177ms\tremaining: 664ms\n",
      "30:\tlearn: 0.1156980\ttest: 0.1124291\tbest: 0.1124291 (30)\ttotal: 221ms\tremaining: 492ms\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "40:\tlearn: 0.0980574\ttest: 0.0947730\tbest: 0.0947730 (40)\ttotal: 265ms\tremaining: 381ms\n",
      "50:\tlearn: 0.0920404\ttest: 0.0890359\tbest: 0.0890359 (50)\ttotal: 310ms\tremaining: 298ms\n",
      "60:\tlearn: 0.0898443\ttest: 0.0871308\tbest: 0.0871308 (60)\ttotal: 353ms\tremaining: 226ms\n",
      "70:\tlearn: 0.0888353\ttest: 0.0863192\tbest: 0.0863192 (70)\ttotal: 395ms\tremaining: 162ms\n",
      "80:\tlearn: 0.0882694\ttest: 0.0859731\tbest: 0.0859731 (80)\ttotal: 438ms\tremaining: 103ms\n",
      "90:\tlearn: 0.0878974\ttest: 0.0858086\tbest: 0.0858086 (90)\ttotal: 482ms\tremaining: 47.6ms\n",
      "99:\tlearn: 0.0876242\ttest: 0.0856751\tbest: 0.0856672 (97)\ttotal: 521ms\tremaining: 0us\n",
      "\n",
      "bestTest = 0.08566721955\n",
      "bestIteration = 97\n",
      "\n",
      "Shrink model to first 98 iterations.\n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0.0     1.0     2.0    3.0  4.0\n",
      "0.0  11073.0   224.0    37.0   94.0  0.0\n",
      "1.0     46.0  8054.0     7.0    4.0  0.0\n",
      "2.0     30.0    96.0  1907.0  118.0  0.0\n",
      "3.0     62.0     4.0    27.0  481.0  0.0\n",
      "4.0      8.0     2.0     0.0    4.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9657509650776551\n",
      "Precision total:  0.7196703991990359\n",
      "Recall total:  0.737290387200625\n",
      "F1 total:  0.7265746441494881\n",
      "BACC total:  0.737290387200625\n",
      "MCC total:  0.9427310738311581\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    }
   ],
   "source": [
    "import catboost\n",
    "\n",
    "cat_01 = catboost.CatBoostClassifier(iterations=100, depth=6, learning_rate=0.1, loss_function='MultiClass', custom_metric='Accuracy')\n",
    "\n",
    "# Fit the model\n",
    "cat_01.fit(X_train_01, y_train_01, eval_set=(X_test_01, y_test_01), verbose=10)\n",
    "\n",
    "# Make predictions on the test set\n",
    "preds_cat = cat_01.predict(X_test_01)\n",
    "preds_cat = np.squeeze(preds_cat)\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('catboost', file = f)\n",
    "\n",
    "\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('CONFUSION MATRIX')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "# pred_label = label[ypred]\n",
    "\n",
    "confusion_matrix = pd.crosstab(y_test_01, pred_label,rownames=['Actual ALERT'],colnames = ['Predicted ALERT'], dropna=False).sort_index(axis=0).sort_index(axis=1)\n",
    "all_unique_values = sorted(set(pred_label) | set(y_test_01))\n",
    "z = np.zeros((len(all_unique_values), len(all_unique_values)))\n",
    "rows, cols = confusion_matrix.shape\n",
    "z[:rows, :cols] = confusion_matrix\n",
    "confusion_matrix  = pd.DataFrame(z, columns=all_unique_values, index=all_unique_values)\n",
    "# confusion_matrix.to_csv('Ensemble_conf_matrix.csv')\n",
    "# with open(output_file_name, \"a\") as f:print(confusion_matrix,file=f)\n",
    "print(confusion_matrix)\n",
    "with open(output_file_name, \"a\") as f: print('Confusion Matrix', file = f)\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print(confusion_matrix, file = f)\n",
    "\n",
    "\n",
    "FP = confusion_matrix.sum(axis=0) - np.diag(confusion_matrix)\n",
    "FN = confusion_matrix.sum(axis=1) - np.diag(confusion_matrix)\n",
    "TP = np.diag(confusion_matrix)\n",
    "TN = confusion_matrix.values.sum() - (FP + FN + TP)\n",
    "TP_total = sum(TP)\n",
    "TN_total = sum(TN)\n",
    "FP_total = sum(FP)\n",
    "FN_total = sum(FN)\n",
    "\n",
    "TP_total = np.array(TP_total,dtype=np.float64)\n",
    "TN_total = np.array(TN_total,dtype=np.float64)\n",
    "FP_total = np.array(FP_total,dtype=np.float64)\n",
    "FN_total = np.array(FN_total,dtype=np.float64)\n",
    "\n",
    "\n",
    "\n",
    "#----------------------------------------------------------------#----------------------------------------------------------------\n",
    "\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('METRICS')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "Acc = accuracy_score(y_test_01, pred_label)\n",
    "Precision = precision_score(y_test_01, pred_label, average='macro')\n",
    "Recall = recall_score(y_test_01, pred_label, average='macro')\n",
    "F1 =  f1_score(y_test_01, pred_label, average='macro')\n",
    "BACC = balanced_accuracy_score(y_test_01, pred_label)\n",
    "MCC = matthews_corrcoef(y_test_01, pred_label)\n",
    "\n",
    "\n",
    "cat_acc_01 = Acc\n",
    "cat_pre_01 = Precision\n",
    "cat_rec_01 = Recall\n",
    "cat_f1_01 = F1\n",
    "cat_bacc_01 = BACC\n",
    "cat_mcc_01 = MCC\n",
    "\n",
    "# with open(output_file_name, \"a\") as f:print('Accuracy total: ', Acc,file=f)\n",
    "print('Accuracy total: ', Acc)\n",
    "cat_acc_01 = Acc\n",
    "print('Precision total: ', Precision )\n",
    "print('Recall total: ', Recall )\n",
    "print('F1 total: ', F1 )\n",
    "print('BACC total: ', BACC)\n",
    "print('MCC total: ', MCC)\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('Accuracy total: ', Acc, file = f)\n",
    "with open(output_file_name, \"a\") as f: print('Precision total: ', Precision, file = f)\n",
    "with open(output_file_name, \"a\") as f: print('Recall total: ', Recall , file = f)\n",
    "with open(output_file_name, \"a\") as f: print('F1 total: ', F1, file = f)\n",
    "with open(output_file_name, \"a\") as f: print('BACC total: ', BACC , file = f)\n",
    "with open(output_file_name, \"a\") as f: print('MCC total: ', MCC, file = f)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0.0     1.0     2.0    3.0  4.0\n",
      "0.0  11213.0    70.0    54.0   91.0  0.0\n",
      "1.0     49.0  8054.0     4.0    4.0  0.0\n",
      "2.0     45.0     5.0  2097.0    4.0  0.0\n",
      "3.0     66.0     1.0     8.0  499.0  0.0\n",
      "4.0     10.0     0.0     0.0    4.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9813717568902056\n",
      "Precision total:  0.7548215666671336\n",
      "Recall total:  0.7636784884768548\n",
      "F1 total:  0.7591512142163392\n",
      "BACC total:  0.7636784884768548\n",
      "MCC total:  0.968696195385652\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "import xgboost as xgb\n",
    "\n",
    "# Create a DMatrix for XGBoost\n",
    "dtrain = xgb.DMatrix(X_train_01, label=y_train_01)\n",
    "dtest = xgb.DMatrix(X_test_01, label=y_test_01)\n",
    "\n",
    "# Set XGBoost parameters\n",
    "params = {\n",
    "    'objective': 'multi:softmax',  # for multi-class classification\n",
    "    'num_class': 5,  # specify the number of classes\n",
    "    'max_depth': 3,\n",
    "    'learning_rate': 0.1,\n",
    "    'eval_metric': 'mlogloss'  # metric for multi-class classification\n",
    "}\n",
    "\n",
    "# Train the XGBoost model\n",
    "num_round = 100\n",
    "xgb_01 = xgb.train(params, dtrain, num_round)\n",
    "\n",
    "# Make predictions on the test set\n",
    "preds_xgb_01 = xgb_01.predict(dtest)\n",
    "\n",
    "\n",
    "if 1 == 1:\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('xgboost base model', file = f)\n",
    "\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('CONFUSION MATRIX')\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "    pred_label = preds_xgb_01\n",
    "    # pred_label = label[ypred]\n",
    "\n",
    "    confusion_matrix = pd.crosstab(y_test_01, pred_label,rownames=['Actual ALERT'],colnames = ['Predicted ALERT'], dropna=False).sort_index(axis=0).sort_index(axis=1)\n",
    "    all_unique_values = sorted(set(pred_label) | set(y_test_01))\n",
    "    z = np.zeros((len(all_unique_values), len(all_unique_values)))\n",
    "    rows, cols = confusion_matrix.shape\n",
    "    z[:rows, :cols] = confusion_matrix\n",
    "    confusion_matrix  = pd.DataFrame(z, columns=all_unique_values, index=all_unique_values)\n",
    "    # confusion_matrix.to_csv('Ensemble_conf_matrix.csv')\n",
    "    # with open(output_file_name, \"a\") as f:print(confusion_matrix,file=f)\n",
    "    print(confusion_matrix)\n",
    "    with open(output_file_name, \"a\") as f: print('Confusion Matrix', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print(confusion_matrix, file = f)\n",
    "\n",
    "\n",
    "    FP = confusion_matrix.sum(axis=0) - np.diag(confusion_matrix)\n",
    "    FN = confusion_matrix.sum(axis=1) - np.diag(confusion_matrix)\n",
    "    TP = np.diag(confusion_matrix)\n",
    "    TN = confusion_matrix.values.sum() - (FP + FN + TP)\n",
    "    TP_total = sum(TP)\n",
    "    TN_total = sum(TN)\n",
    "    FP_total = sum(FP)\n",
    "    FN_total = sum(FN)\n",
    "\n",
    "    TP_total = np.array(TP_total,dtype=np.float64)\n",
    "    TN_total = np.array(TN_total,dtype=np.float64)\n",
    "    FP_total = np.array(FP_total,dtype=np.float64)\n",
    "    FN_total = np.array(FN_total,dtype=np.float64)\n",
    "\n",
    "\n",
    "\n",
    "    #----------------------------------------------------------------#----------------------------------------------------------------\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('METRICS')\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "    # Acc = ACC(TP_total,TN_total, FP_total, FN_total)\n",
    "    # Precision = PRECISION(TP_total, FP_total)\n",
    "    # Recall = RECALL(TP_total, FN_total)\n",
    "    # F1 = F1(Recall,Precision)\n",
    "    # BACC = BACC(TP_total,TN_total, FP_total, FN_total)\n",
    "    # MCC = MCC(TP_total,TN_total, FP_total, FN_total)\n",
    "\n",
    "    Acc = accuracy_score(y_test_01, pred_label)\n",
    "    Precision = precision_score(y_test_01, pred_label, average='macro')\n",
    "    Recall = recall_score(y_test_01, pred_label, average='macro')\n",
    "    F1 =  f1_score(y_test_01, pred_label, average='macro')\n",
    "    BACC = balanced_accuracy_score(y_test_01, pred_label)\n",
    "    MCC = matthews_corrcoef(y_test_01, pred_label)\n",
    "\n",
    "    xgb_acc_01 = Acc\n",
    "    xgb_pre_01 = Precision\n",
    "    xgb_rec_01 = Recall\n",
    "    xgb_f1_01 = F1\n",
    "    xgb_bacc_01 = BACC\n",
    "    xgb_mcc_01 = MCC\n",
    "\n",
    "\n",
    "    # with open(output_file_name, \"a\") as f:print('Accuracy total: ', Acc,file=f)\n",
    "    print('Accuracy total: ', Acc)\n",
    "    print('Precision total: ', Precision )\n",
    "    print('Recall total: ', Recall )\n",
    "    print('F1 total: ', F1 )\n",
    "    print('BACC total: ', BACC)\n",
    "    print('MCC total: ', MCC)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Accuracy total: ', Acc, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Precision total: ', Precision, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Recall total: ', Recall , file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('F1 total: ', F1, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('BACC total: ', BACC , file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('MCC total: ', MCC, file = f)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model1 = tree.DecisionTreeClassifier()\n",
    "# model2 = KNeighborsClassifier()\n",
    "# model3= LogisticRegression()\n",
    "\n",
    "# model1.fit(x_train,y_train)\n",
    "# model2.fit(x_train,y_train)\n",
    "# model3.fit(x_train,y_train)\n",
    "\n",
    "# pred1=model1.predict_proba(x_test)\n",
    "# pred2=model2.predict_proba(x_test)\n",
    "# pred3=model3.predict_proba(x_test)\n",
    "\n",
    "# finalpred=(preds_svm_prob +\n",
    "#             preds_ada_prob +\n",
    "#             preds_knn_prob +\n",
    "#             preds_rf_prob +\n",
    "#             preds_dnn_prob +\n",
    "#             preds_lgbm_prob +\n",
    "#             preds_mlp_prob\n",
    "#             )/7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "# with open(output_file_name, \"a\") as f: print('-----------------------', file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Summary', file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('-----------------------', file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Level 00', file = f)\n",
    "\n",
    "# with open(output_file_name, \"a\") as f: print('Accuracy ada: ', ada_acc_00, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Accuracy dnn: ', dnn_acc_00, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Accuracy svm: ', svm_acc_00, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Accuracy knn: ', knn_acc_00, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Accuracy mlp: ', mlp_acc_00, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Accuracy lgbm: ', lgbm_acc_00, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Accuracy rf: ', rf_acc_00, file = f)\n",
    "\n",
    "# with open(output_file_name, \"a\") as f: print('-----------------------', file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Level 01', file = f)\n",
    "\n",
    "# with open(output_file_name, \"a\") as f: print('Accuracy ada: ', ada_acc_01, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Accuracy dnn: ', dnn_acc_01, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Accuracy svm: ', svm_acc_01, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Accuracy knn: ', knn_acc_01, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Accuracy mlp: ', mlp_acc_01, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Accuracy lgbm: ', lgbm_acc_01, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Accuracy rf: ', rf_acc_01, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Accuracy LR: ', lr_acc_01, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Accuracy Voting: ', voring_acc, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Accuracy catboost: ', cat_acc_01, file = f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "# with open(output_file_name, \"a\") as f: print('-----------------------', file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Summary', file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('-----------------------', file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Level 00', file = f)\n",
    "\n",
    "# with open(output_file_name, \"a\") as f: print('Accuracy ada: ', ada_acc_00, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Accuracy dnn: ', dnn_acc_00, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Accuracy svm: ', svm_acc_00, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Accuracy knn: ', knn_acc_00, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Accuracy mlp: ', mlp_acc_00, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Accuracy lgbm: ', lgbm_acc_00, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Accuracy rf: ', rf_acc_00, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Accuracy cat: ', cat_acc_00, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Accuracy xgb: ', xgb_acc_00, file = f)\n",
    "\n",
    "\n",
    "# with open(output_file_name, \"a\") as f: print('Precision ada: ', ada_pre_00, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Precision dnn: ', dnn_pre_00, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Precision svm: ', svm_pre_00, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Precision knn: ', knn_pre_00, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Precision mlp: ', mlp_pre_00, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Precision lgbm: ', lgbm_pre_00, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Precision rf: ', rf_pre_00, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Precision cat: ', cat_pre_00, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Precision xgb: ', xgb_pre_00, file = f)\n",
    "\n",
    "# with open(output_file_name, \"a\") as f: print('Recall ada: ', ada_rec_00, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Recall dnn: ', dnn_rec_00, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Recall svm: ', svm_rec_00, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Recall knn: ', knn_rec_00, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Recall mlp: ', mlp_rec_00, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Recall lgbm: ', lgbm_rec_00, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Recall rf: ', rf_rec_00, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Recall cat: ', cat_rec_00, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Recall xgb: ', xgb_rec_00, file = f)\n",
    "\n",
    "# with open(output_file_name, \"a\") as f: print('F1 ada: ', ada_f1_00, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('F1 dnn: ', dnn_f1_00, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('F1 svm: ', svm_f1_00, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('F1 knn: ', knn_f1_00, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('F1 mlp: ', mlp_f1_00, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('F1 lgbm: ', lgbm_f1_00, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('F1 rf: ', rf_f1_00, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('F1 cat: ', cat_f1_00, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('F1 xgb: ', xgb_f1_00, file = f)\n",
    "\n",
    "\n",
    "# with open(output_file_name, \"a\") as f: print('-----------------------', file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Level 01', file = f)\n",
    "\n",
    "# with open(output_file_name, \"a\") as f: print('Accuracy ada: ', ada_acc_01, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Accuracy dnn: ', dnn_acc_01, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Accuracy svm: ', svm_acc_01, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Accuracy knn: ', knn_acc_01, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Accuracy mlp: ', mlp_acc_01, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Accuracy lgbm: ', lgbm_acc_01, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Accuracy rf: ', rf_acc_01, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Accuracy LR: ', lr_acc_01, file = f)\n",
    "# # with open(output_file_name, \"a\") as f: print('Accuracy Voting: ', voting_acc_01, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Accuracy catboost: ', cat_acc_01, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Accuracy xgb: ', xgb_acc_01, file = f)\n",
    "\n",
    "# with open(output_file_name, \"a\") as f: print('Precision ada: ', ada_pre_01, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Precision dnn: ', dnn_pre_01, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Precision svm: ', svm_pre_01, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Precision knn: ', knn_pre_01, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Precision mlp: ', mlp_pre_01, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Precision lgbm: ', lgbm_pre_01, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Precision rf: ', rf_pre_01, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Precision LR: ', lr_pre_01, file = f)\n",
    "# # with open(output_file_name, \"a\") as f: print('Precision Voting: ', voting_pre_01, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Precision catboosting: ', cat_pre_01, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Precision xgboost: ', xgb_pre_01, file = f)\n",
    "\n",
    "# with open(output_file_name, \"a\") as f: print('Recall ada: ', ada_rec_01, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Recall dnn: ', dnn_rec_01, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Recall svm: ', svm_rec_01, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Recall knn: ', knn_rec_01, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Recall mlp: ', mlp_rec_01, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Recall lgbm: ', lgbm_rec_01, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Recall rf: ', rf_rec_01, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Recall LR: ', lr_rec_01, file = f)\n",
    "# # with open(output_file_name, \"a\") as f: print('Recall Voting: ', voting_rec_01, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Recall catboosting: ', cat_rec_01, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Recall xgboost: ', xgb_rec_01, file = f)\n",
    "\n",
    "# with open(output_file_name, \"a\") as f: print('F1 ada: ', ada_f1_01, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('F1 dnn: ', dnn_f1_01, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('F1 svm: ', svm_f1_01, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('F1 knn: ', knn_f1_01, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('F1 mlp: ', mlp_f1_01, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('F1 lgbm: ', lgbm_f1_01, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('F1 rf: ', rf_f1_01, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('F1 LR: ', lr_f1_01, file = f)\n",
    "# # with open(output_file_name, \"a\") as f: print('F1 Voting: ', voting_f1_01, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('F1 catboosting: ', cat_f1_01, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('F1 xgboost: ', xgb_f1_01, file = f)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "# with open(output_file_name, \"a\") as f: print('-----------------------', file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Summary', file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('-----------------------', file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Level 00', file = f)\n",
    "\n",
    "# with open(output_file_name, \"a\") as f: print('Accuracy ada: ', ada_acc_00, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Accuracy dnn: ', dnn_acc_00, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Accuracy svm: ', svm_acc_00, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Accuracy knn: ', knn_acc_00, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Accuracy mlp: ', mlp_acc_00, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Accuracy lgbm: ', lgbm_acc_00, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Accuracy rf: ', rf_acc_00, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Accuracy cat: ', cat_acc_00, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Accuracy xgb: ', xgb_acc_00, file = f)\n",
    "\n",
    "\n",
    "# # with open(output_file_name, \"a\") as f: print('Precision ada: ', ada_pre_00, file = f)\n",
    "# # with open(output_file_name, \"a\") as f: print('Precision dnn: ', dnn_pre_00, file = f)\n",
    "# # with open(output_file_name, \"a\") as f: print('Precision svm: ', svm_pre_00, file = f)\n",
    "# # with open(output_file_name, \"a\") as f: print('Precision knn: ', knn_pre_00, file = f)\n",
    "# # with open(output_file_name, \"a\") as f: print('Precision mlp: ', mlp_pre_00, file = f)\n",
    "# # with open(output_file_name, \"a\") as f: print('Precision lgbm: ', lgbm_pre_00, file = f)\n",
    "# # with open(output_file_name, \"a\") as f: print('Precision rf: ', rf_pre_00, file = f)\n",
    "# # with open(output_file_name, \"a\") as f: print('Precision cat: ', cat_pre_00, file = f)\n",
    "# # with open(output_file_name, \"a\") as f: print('Precision xgb: ', xgb_pre_00, file = f)\n",
    "\n",
    "# # with open(output_file_name, \"a\") as f: print('Recall ada: ', ada_rec_00, file = f)\n",
    "# # with open(output_file_name, \"a\") as f: print('Recall dnn: ', dnn_rec_00, file = f)\n",
    "# # with open(output_file_name, \"a\") as f: print('Recall svm: ', svm_rec_00, file = f)\n",
    "# # with open(output_file_name, \"a\") as f: print('Recall knn: ', knn_rec_00, file = f)\n",
    "# # with open(output_file_name, \"a\") as f: print('Recall mlp: ', mlp_rec_00, file = f)\n",
    "# # with open(output_file_name, \"a\") as f: print('Recall lgbm: ', lgbm_rec_00, file = f)\n",
    "# # with open(output_file_name, \"a\") as f: print('Recall rf: ', rf_rec_00, file = f)\n",
    "# # with open(output_file_name, \"a\") as f: print('Recall cat: ', cat_rec_00, file = f)\n",
    "# # with open(output_file_name, \"a\") as f: print('Recall xgb: ', xgb_rec_00, file = f)\n",
    "\n",
    "# # with open(output_file_name, \"a\") as f: print('F1 ada: ', ada_f1_00, file = f)\n",
    "# # with open(output_file_name, \"a\") as f: print('F1 dnn: ', dnn_f1_00, file = f)\n",
    "# # with open(output_file_name, \"a\") as f: print('F1 svm: ', svm_f1_00, file = f)\n",
    "# # with open(output_file_name, \"a\") as f: print('F1 knn: ', knn_f1_00, file = f)\n",
    "# # with open(output_file_name, \"a\") as f: print('F1 mlp: ', mlp_f1_00, file = f)\n",
    "# # with open(output_file_name, \"a\") as f: print('F1 lgbm: ', lgbm_f1_00, file = f)\n",
    "# # with open(output_file_name, \"a\") as f: print('F1 rf: ', rf_f1_00, file = f)\n",
    "# # with open(output_file_name, \"a\") as f: print('F1 cat: ', cat_f1_00, file = f)\n",
    "# # with open(output_file_name, \"a\") as f: print('F1 xgb: ', xgb_f1_00, file = f)\n",
    "\n",
    "\n",
    "# with open(output_file_name, \"a\") as f: print('-----------------------', file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Level 01', file = f)\n",
    "\n",
    "# with open(output_file_name, \"a\") as f: print('Accuracy ada: ', ada_acc_01, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Accuracy dnn: ', dnn_acc_01, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Accuracy svm: ', svm_acc_01, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Accuracy knn: ', knn_acc_01, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Accuracy mlp: ', mlp_acc_01, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Accuracy lgbm: ', lgbm_acc_01, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Accuracy rf: ', rf_acc_01, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Accuracy LR: ', lr_acc_01, file = f)\n",
    "# # with open(output_file_name, \"a\") as f: print('Accuracy Voting: ', voting_acc, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Accuracy catboost: ', cat_acc_01, file = f)\n",
    "# with open(output_file_name, \"a\") as f: print('Accuracy xgb: ', xgb_acc_01, file = f)\n",
    "\n",
    "# # with open(output_file_name, \"a\") as f: print('Precision ada: ', ada_pre_01, file = f)\n",
    "# # with open(output_file_name, \"a\") as f: print('Precision dnn: ', dnn_pre_01, file = f)\n",
    "# # with open(output_file_name, \"a\") as f: print('Precision svm: ', svm_pre_01, file = f)\n",
    "# # with open(output_file_name, \"a\") as f: print('Precision knn: ', knn_pre_01, file = f)\n",
    "# # with open(output_file_name, \"a\") as f: print('Precision mlp: ', mlp_pre_01, file = f)\n",
    "# # with open(output_file_name, \"a\") as f: print('Precision lgbm: ', lgbm_pre_01, file = f)\n",
    "# # with open(output_file_name, \"a\") as f: print('Precision rf: ', rf_pre_01, file = f)\n",
    "# # with open(output_file_name, \"a\") as f: print('Precision LR: ', lr_pre_01, file = f)\n",
    "# # with open(output_file_name, \"a\") as f: print('Precision Voting: ', voting_pre, file = f)\n",
    "# # with open(output_file_name, \"a\") as f: print('Precision catboosting: ', cat_pre_01, file = f)\n",
    "# # with open(output_file_name, \"a\") as f: print('Precision xgboost: ', xgb_pre_01, file = f)\n",
    "\n",
    "# # with open(output_file_name, \"a\") as f: print('Recall ada: ', ada_rec_01, file = f)\n",
    "# # with open(output_file_name, \"a\") as f: print('Recall dnn: ', dnn_rec_01, file = f)\n",
    "# # with open(output_file_name, \"a\") as f: print('Recall svm: ', svm_rec_01, file = f)\n",
    "# # with open(output_file_name, \"a\") as f: print('Recall knn: ', knn_rec_01, file = f)\n",
    "# # with open(output_file_name, \"a\") as f: print('Recall mlp: ', mlp_rec_01, file = f)\n",
    "# # with open(output_file_name, \"a\") as f: print('Recall lgbm: ', lgbm_rec_01, file = f)\n",
    "# # with open(output_file_name, \"a\") as f: print('Recall rf: ', rf_rec_01, file = f)\n",
    "# # with open(output_file_name, \"a\") as f: print('Recall LR: ', lr_rec_01, file = f)\n",
    "# # with open(output_file_name, \"a\") as f: print('Recall Voting: ', voting_rec, file = f)\n",
    "# # with open(output_file_name, \"a\") as f: print('Recall catboosting: ', cat_rec_01, file = f)\n",
    "# # with open(output_file_name, \"a\") as f: print('Recall xgboost: ', xgb_rec_01, file = f)\n",
    "\n",
    "# # with open(output_file_name, \"a\") as f: print('F1 ada: ', ada_f1_01, file = f)\n",
    "# # with open(output_file_name, \"a\") as f: print('F1 dnn: ', dnn_f1_01, file = f)\n",
    "# # with open(output_file_name, \"a\") as f: print('F1 svm: ', svm_f1_01, file = f)\n",
    "# # with open(output_file_name, \"a\") as f: print('F1 knn: ', knn_f1_01, file = f)\n",
    "# # with open(output_file_name, \"a\") as f: print('F1 mlp: ', mlp_f1_01, file = f)\n",
    "# # with open(output_file_name, \"a\") as f: print('F1 lgbm: ', lgbm_f1_01, file = f)\n",
    "# # with open(output_file_name, \"a\") as f: print('F1 rf: ', rf_f1_01, file = f)\n",
    "# # with open(output_file_name, \"a\") as f: print('F1 LR: ', lr_f1_01, file = f)\n",
    "# # with open(output_file_name, \"a\") as f: print('F1 Voting: ', voting_f1, file = f)\n",
    "# # with open(output_file_name, \"a\") as f: print('F1 catboosting: ', cat_f1_01, file = f)\n",
    "# # with open(output_file_name, \"a\") as f: print('F1 xgboost: ', xgb_f1_01, file = f)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# import sklearn\n",
    "# from sklearn.model_selection import train_test_split\n",
    "# split = 0.7\n",
    "\n",
    "# #AUC ROC\n",
    "# #---------------------------------------------------------------------\n",
    "\n",
    "# #AUCROC\n",
    "# aucroc =[]\n",
    "# y_array = [y_0,y_1,y_2,y_3,y_4]\n",
    "# for j in range(0,len(y_array)):\n",
    "#     # print(j)\n",
    "#     #------------------------------------------------------------------------------------------------------------\n",
    "#     X_train,X_test, y_train, y_test = sklearn.model_selection.train_test_split(X, y_array[j], train_size=split)\n",
    "    \n",
    "#     # evaluate the model\n",
    "\n",
    "#     knn_clf.fit(X_train,y_train)\n",
    "#     y_pred=knn_clf.predict(X_test) #These are the predicted output value\n",
    "#     # y_pred = knn_clf.predict_proba(X_test)\n",
    "\n",
    "    \n",
    "#     y_scores = y_pred\n",
    "#     y_true = y_test\n",
    "\n",
    "#     # model = LGBMClassifier()\n",
    "#     # model.fit(X_train, y_train)\n",
    "#     # y_pred = model.predict(X_test)\n",
    "\n",
    "\n",
    "#     y_scores = y_pred\n",
    "#     y_true = y_test\n",
    "    \n",
    "#     # Calculate AUC-ROC score\n",
    "#     auc_roc_score= roc_auc_score(y_true, y_scores,  average='weighted')  # Use 'micro' or 'macro' for different averaging strategies\n",
    "#     # print(\"AUC-ROC Score class:\", auc_roc_score)\n",
    "#     aucroc.append(auc_roc_score)\n",
    "#     #-------------------------------------------------------------------------------------------------------    -----\n",
    "#     # Calculate the average\n",
    "# average = sum(aucroc) / len(aucroc)\n",
    "\n",
    "# # Display the result\n",
    "# # with open(output_file_name, \"a\") as f:print(\"AUC ROC Average:\", average, file = f)\n",
    "# print(\"AUC ROC Average:\", average)\n",
    "\n",
    "# #End AUC ROC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "# lr_acc_00 = 0 \n",
    "# voting_acc_00 = 0\n",
    "\n",
    "# lr_pre_00 = 0 \n",
    "# voting_pre_00 = 0\n",
    "\n",
    "# lr_rec_00 = 0 \n",
    "# voting_rec_00 = 0\n",
    "\n",
    "# lr_f1_00 = 0 \n",
    "# voting_f1_00 = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "voting_acc_01 = 0\n",
    "\n",
    "\n",
    "voting_pre_01 = 0\n",
    "\n",
    "\n",
    "voting_rec_01 = 0\n",
    "\n",
    "\n",
    "voting_f1_01 = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "| Models   | ACC-01             | PRE-01             | REC-01             | F1-01              |\n",
      "+==========+====================+====================+====================+====================+\n",
      "| ADA      | 0.9667833737319328 | 0.7452786169091856 | 0.7069300802727743 | 0.724398988880189  |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "| SVM      | 0.8792081874495018 | 0.5783299354191134 | 0.5599734594264867 | 0.5135936316789838 |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "| DNN      | 0.9430379746835443 | 0.5325704376912934 | 0.5579176145445526 | 0.5446612807463953 |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "| MLP      | 0.9811473202262322 | 0.7540175171198078 | 0.7632528959239488 | 0.7585292769243145 |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "| KNN      | 0.979890474907981  | 0.7553148767605853 | 0.7583261670574462 | 0.7567766366742548 |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "| CAT      | 0.9657509650776551 | 0.7196703991990359 | 0.737290387200625  | 0.7265746441494881 |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "| XGB      | 0.9813717568902056 | 0.7548215666671336 | 0.7636784884768548 | 0.7591512142163392 |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "| LGBM     | 0.9812819822246163 | 0.7547768376013453 | 0.7635608505955311 | 0.7590703045266477 |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "| RF       | 0.9811473202262322 | 0.7535358004758851 | 0.7642242184751191 | 0.7587357911044557 |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "| LR       | 0.9657509650776551 | 0.7196703991990359 | 0.737290387200625  | 0.7265746441494881 |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "| VOTING   | 0                  | 0                  | 0                  | 0                  |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "|          |                    |                    |                    |                    |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n"
     ]
    }
   ],
   "source": [
    "from tabulate import tabulate\n",
    "\n",
    "# Assuming data is a 110x4 list, where each row is a sublist\n",
    "# data =  [[\"Row {} Col {}\".format(i + 1, j + 1) for j in range(4)] for i in range(110)]\n",
    "data = [[\"\" for _ in range(5)] for _ in range(12)]\n",
    "\n",
    "# Manually insert data at specific row and column\n",
    "# data[0][0] = \"ADA\"\n",
    "# data[1][0] = \"DNN\"\n",
    "# data[2][0] = \"SVM\"\n",
    "# data[3][0] = \"ADA\"\n",
    "# data[4][0] = \"DNN\"\n",
    "# data[2][0] = \"SVM\"\n",
    "\n",
    "\n",
    "names_models = ['ADA',\n",
    "                'SVM',\n",
    "                'DNN',\n",
    "                'MLP',\n",
    "                'KNN',\n",
    "                'CAT',\n",
    "                'XGB',\n",
    "                'LGBM',\n",
    "                'RF',\n",
    "                'LR',\n",
    "                'VOTING'\n",
    "                ]\n",
    "level_01_acc = [ada_acc_01,\n",
    "                svm_acc_01,\n",
    "                dnn_acc_01,\n",
    "                mlp_acc_01,\n",
    "                knn_acc_01,\n",
    "                cat_acc_01,\n",
    "                xgb_acc_01,\n",
    "                lgbm_acc_01,\n",
    "                rf_acc_01,\n",
    "                lr_acc_01,\n",
    "                voting_acc_01]  \n",
    "level_01_pre = [ada_pre_01,\n",
    "                svm_pre_01,\n",
    "                dnn_pre_01,\n",
    "                mlp_pre_01,\n",
    "                knn_pre_01,\n",
    "                cat_pre_01,\n",
    "                xgb_pre_01,\n",
    "                lgbm_pre_01,\n",
    "                rf_pre_01,\n",
    "                lr_pre_01,\n",
    "                voting_pre_01] \n",
    "level_01_rec = [ada_rec_01,\n",
    "                svm_rec_01,\n",
    "                dnn_rec_01,\n",
    "                mlp_rec_01,\n",
    "                knn_rec_01,\n",
    "                cat_rec_01,\n",
    "                xgb_rec_01,\n",
    "                lgbm_rec_01,\n",
    "                rf_rec_01,\n",
    "                lr_rec_01,\n",
    "                voting_rec_01]  \n",
    "level_01_f1 = [ada_f1_01,\n",
    "                svm_f1_01,\n",
    "                dnn_f1_01,\n",
    "                mlp_f1_01,\n",
    "                knn_f1_01,\n",
    "                cat_f1_01,\n",
    "                xgb_f1_01,\n",
    "                lgbm_f1_01,\n",
    "                rf_f1_01,\n",
    "                lr_f1_01,\n",
    "                voting_f1_01]  \n",
    "\n",
    "                 \n",
    "\n",
    "for i in range(0,len(names_models)):\n",
    "    data[i][0] =  names_models[i]\n",
    "\n",
    "    data[i][1] = level_01_acc[i]\n",
    "    # data[i][2] = level_01_acc[i]\n",
    "\n",
    "    data[i][2] = level_01_pre[i] \n",
    "    # data[i][4] = level_01_pre[i]\n",
    "\n",
    "    data[i][3] = level_01_rec[i] \n",
    "    # data[i][6] = level_01_rec[i]\n",
    "\n",
    "    data[i][4] = level_01_f1[i]\n",
    "    # data[i][8] = level_01_f1[i]\n",
    "\n",
    "\n",
    "\n",
    "\n",
    " \n",
    "# data[0][1] = ada_acc_01\n",
    "# data\n",
    "\n",
    "# Define column headers\n",
    "# headers = [\"Models\", \"ACC-01\", \" ACC-01\",\"PRE-01\", \" PRE-01\",\"REC-01\", \" REC-01\",\"F1-01\", \" F1-01\",]\n",
    "headers = [\"Models\", \"ACC-01\",\"PRE-01\",\"REC-01\",\"F1-01\"]\n",
    "\n",
    "\n",
    "# Print the table\n",
    "table = tabulate(data, headers=headers, tablefmt=\"grid\")\n",
    "print(table)\n",
    "with open(output_file_name, \"a\") as f: print(table, file = f)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using 250 background data samples could cause slower run times. Consider using shap.sample(data, K) or shap.kmeans(data, K) to summarize the background as K samples.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Generating SHAP explanation\n",
      "---------------------------------------------------------------------------------\n",
      "\n",
      "---------------------------------------------------------------------------------\n",
      "Generating explainer\n",
      "---------------------------------------------------------------------------------\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  3%|▎         | 8/250 [00:00<00:14, 17.21it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 250/250 [00:13<00:00, 18.39it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  col_name  feature_importance_vals\n",
      "2     lgbm                 0.034430\n",
      "1       rf                 0.011712\n",
      "3      xgb                 0.007331\n",
      "0      dnn                 0.006204\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 576x223.2 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print('---------------------------------------------------------------------------------')\n",
    "print('Generating SHAP explanation')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('')\n",
    "with open(output_file_name, \"a\") as f:print('ADA FEATURE IMPORTANCE',file = f)\n",
    "\n",
    "    #START TIMER MODEL\n",
    "start = time.time()\n",
    "\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Generating explainer')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('')\n",
    "test = X_test_01\n",
    "train = X_train_01\n",
    "# ## Summary Bar Plot Global\n",
    "start_index = 0\n",
    "end_index = 250\n",
    "# test.pop('Label')\n",
    "# test.pop('is_train')\n",
    "# print(label2)\n",
    "\n",
    "\n",
    "# models = [ada,dnn_01,clf,knn_clf_01,cat_01,xgb_01, rf, lgbm, mlp,logreg_01]\n",
    "explainer = shap.KernelExplainer(ada.predict_proba, test[start_index:end_index])\n",
    "\n",
    "shap_values = explainer.shap_values(test[start_index:end_index])\n",
    "\n",
    "shap.summary_plot(shap_values = shap_values,\n",
    "                  features = test[start_index:end_index],\n",
    "                  # class_names=[column_features[:-1]],\n",
    "                  show=False)\n",
    "\n",
    "# if feature_selection_bit == 1 # On\n",
    "# pick_prob = 0 # set equal one to choose the dataset with probabilities, set to 0 to choose one with the classes.\n",
    "if pick_prob == 1:\n",
    "  plt.savefig('ADA_SHAP_NSL_prob_01.png')\n",
    "elif pick_prob == 0:\n",
    "  plt.savefig('ADA_SHAP_NSL_class_01.png')\n",
    "\n",
    "else: None\n",
    "plt.clf()\n",
    "\n",
    "\n",
    "vals= np.abs(shap_values).mean(1)\n",
    "feature_importance = pd.DataFrame(list(zip(train.columns, sum(vals))), columns=['col_name','feature_importance_vals'])\n",
    "feature_importance.sort_values(by=['feature_importance_vals'], ascending=False,inplace=True)\n",
    "feature_importance.head()\n",
    "print(feature_importance.to_string())\n",
    "\n",
    "with open(output_file_name, \"a\") as f:print('Feature Importance: ',feature_importance.to_string(),file = f)\n",
    "\n",
    "\n",
    "\n",
    "end = time.time()\n",
    "with open(output_file_name, \"a\") as f:print('ELAPSE TIME LIME GLOBAL: ',(end - start)/60, 'min',file = f)\n",
    "print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "\n",
    "print('---------------------------------------------------------------------------------')\n",
    "# feature_importance_vals = 'feature_importance_vals'  # Replace with the name of the column you want to extract\n",
    "feature_val = feature_importance['feature_importance_vals'].tolist()\n",
    "\n",
    "# col_name = 'col_name'  # Replace with the name of the column you want to extract\n",
    "feature_name = feature_importance['col_name'].tolist()\n",
    "\n",
    "\n",
    "# for item1, item2 in zip(feature_name, feature_val):\n",
    "#     print(item1, item2)\n",
    "\n",
    "\n",
    "# Use zip to combine the two lists, sort based on list1, and then unzip them\n",
    "zipped_lists = list(zip(feature_name, feature_val))\n",
    "zipped_lists.sort(key=lambda x: x[1],reverse=True)\n",
    "\n",
    "# Convert the sorted result back into separate lists\n",
    "sorted_list1, sorted_list2 = [list(x) for x in zip(*zipped_lists)]\n",
    "\n",
    "# for k in sorted_list1:\n",
    "#   with open(output_file_name, \"a\") as f: print(\"df.pop('\",k,\"')\", sep='', file = f)\n",
    "\n",
    "# with open(output_file_name, \"a\") as f:print(\"Trial_ =[\", file = f)\n",
    "# for k in sorted_list1:\n",
    "#   with open(output_file_name, \"a\") as f:print(\"'\",k,\"',\", sep='', file = f)\n",
    "# with open(output_file_name, \"a\") as f:print(\"]\", file = f)\n",
    "\n",
    "print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# explainer = shap.TreeExplainer(model)\n",
    "# start_index = 0\n",
    "# end_index = samples\n",
    "# shap_values = explainer.shap_values(test[start_index:end_index])\n",
    "# shap_obj = explainer(test[start_index:end_index])\n",
    "# shap.summary_plot(shap_values = shap_values,\n",
    "#                   features = test[start_index:end_index],\n",
    "#                 show=False)\n",
    "# plt.savefig('Light_SHAP_CIC_Summary.png')\n",
    "# plt.clf()\n",
    "\n",
    "\n",
    "# vals= np.abs(shap_values).mean(1)\n",
    "# feature_importance = pd.DataFrame(list(zip(train.columns, sum(vals))), columns=['col_name','feature_importance_vals'])\n",
    "# feature_importance.sort_values(by=['feature_importance_vals'], ascending=False,inplace=True)\n",
    "# feature_importance.head()\n",
    "# print(feature_importance.to_string())\n",
    "# print('---------------------------------------------------------------------------------')\n",
    "# # feature_importance_vals = 'feature_importance_vals'  # Replace with the name of the column you want to extract\n",
    "# feature_val = feature_importance['feature_importance_vals'].tolist()\n",
    "\n",
    "# # col_name = 'col_name'  # Replace with the name of the column you want to extract\n",
    "# feature_name = feature_importance['col_name'].tolist()\n",
    "\n",
    "\n",
    "# # for item1, item2 in zip(feature_name, feature_val):\n",
    "# #     print(item1, item2)\n",
    "\n",
    "\n",
    "# # Use zip to combine the two lists, sort based on list1, and then unzip them\n",
    "# zipped_lists = list(zip(feature_name, feature_val))\n",
    "# zipped_lists.sort(key=lambda x: x[1],reverse=True)\n",
    "\n",
    "# # Convert the sorted result back into separate lists\n",
    "# sorted_list1, sorted_list2 = [list(x) for x in zip(*zipped_lists)]\n",
    "\n",
    "# for k in sorted_list1:\n",
    "#   with open(output_file_name, \"a\") as f:print(\"df.pop('\",k,\"')\", sep='', file = f)\n",
    "\n",
    "# # with open(output_file_name, \"a\") as f:print(\"Trial_ =[\", file = f)\n",
    "# for k in sorted_list1:\n",
    "#   with open(output_file_name, \"a\") as f:print(\"'\",k,\"',\", sep='', file = f)\n",
    "# with open(output_file_name, \"a\") as f:print(\"]\", file = f)\n",
    "# print('---------------------------------------------------------------------------------')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Generating SHAP explanation\n",
      "---------------------------------------------------------------------------------\n",
      "\n",
      "---------------------------------------------------------------------------------\n",
      "Generating explainer\n",
      "---------------------------------------------------------------------------------\n",
      "\n",
      "  col_name  feature_importance_vals\n",
      "2     lgbm                 6.412169\n",
      "1       rf                 2.172974\n",
      "3      xgb                 2.030175\n",
      "0      dnn                 1.079601\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 576x223.2 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print('---------------------------------------------------------------------------------')\n",
    "print('Generating SHAP explanation')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('')\n",
    "\n",
    "with open(output_file_name, \"a\") as f:print('XGB FEATURE IMPORTANCE',file = f)\n",
    "\n",
    "    #START TIMER MODEL\n",
    "start = time.time()\n",
    "\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Generating explainer')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('')\n",
    "test = X_test_01\n",
    "train = X_train_01\n",
    "# ## Summary Bar Plot Global\n",
    "start_index = 0\n",
    "end_index = 250\n",
    "# test.pop('Label')\n",
    "# test.pop('is_train')\n",
    "# print(label2)\n",
    "\n",
    "\n",
    "explainer = shap.TreeExplainer(xgb_01)\n",
    "\n",
    "shap_values = explainer.shap_values(test[start_index:end_index])\n",
    "shap_obj = explainer(test[start_index:end_index])\n",
    "shap.summary_plot(shap_values = shap_values,\n",
    "                  features = test[start_index:end_index],\n",
    "                show=False)\n",
    "# plt.clf()\n",
    "\n",
    "# if feature_selection_bit == 1 # On\n",
    "# pick_prob = 0 # set equal one to choose the dataset with probabilities, set to 0 to choose one with the classes.\n",
    "if pick_prob == 1:\n",
    "  plt.savefig('XGB_SHAP_NSL_prob_01.png')\n",
    "elif pick_prob == 0:\n",
    "  plt.savefig('XGB_SHAP_NSL_class_01.png')\n",
    "\n",
    "else: None\n",
    "plt.clf()\n",
    "\n",
    "\n",
    "vals= np.abs(shap_values).mean(1)\n",
    "feature_importance = pd.DataFrame(list(zip(train.columns, sum(vals))), columns=['col_name','feature_importance_vals'])\n",
    "feature_importance.sort_values(by=['feature_importance_vals'], ascending=False,inplace=True)\n",
    "feature_importance.head()\n",
    "print(feature_importance.to_string())\n",
    "with open(output_file_name, \"a\") as f:print('Feature Importance: ',feature_importance.to_string(),file = f)\n",
    "\n",
    "\n",
    "\n",
    "end = time.time()\n",
    "with open(output_file_name, \"a\") as f:print('ELAPSE TIME LIME GLOBAL: ',(end - start)/60, 'min',file = f)\n",
    "print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "\n",
    "print('---------------------------------------------------------------------------------')\n",
    "# feature_importance_vals = 'feature_importance_vals'  # Replace with the name of the column you want to extract\n",
    "feature_val = feature_importance['feature_importance_vals'].tolist()\n",
    "\n",
    "# col_name = 'col_name'  # Replace with the name of the column you want to extract\n",
    "feature_name = feature_importance['col_name'].tolist()\n",
    "\n",
    "\n",
    "# for item1, item2 in zip(feature_name, feature_val):\n",
    "#     print(item1, item2)\n",
    "\n",
    "\n",
    "# Use zip to combine the two lists, sort based on list1, and then unzip them\n",
    "zipped_lists = list(zip(feature_name, feature_val))\n",
    "zipped_lists.sort(key=lambda x: x[1],reverse=True)\n",
    "\n",
    "# Convert the sorted result back into separate lists\n",
    "sorted_list1, sorted_list2 = [list(x) for x in zip(*zipped_lists)]\n",
    "\n",
    "# for k in sorted_list1:\n",
    "#   with open(output_file_name, \"a\") as f: print(\"df.pop('\",k,\"')\", sep='', file = f)\n",
    "\n",
    "# with open(output_file_name, \"a\") as f:print(\"Trial_ =[\", file = f)\n",
    "# for k in sorted_list1:\n",
    "#   with open(output_file_name, \"a\") as f:print(\"'\",k,\"',\", sep='', file = f)\n",
    "# with open(output_file_name, \"a\") as f:print(\"]\", file = f)\n",
    "\n",
    "print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Generating SHAP explanation\n",
      "---------------------------------------------------------------------------------\n",
      "\n",
      "---------------------------------------------------------------------------------\n",
      "Generating explainer\n",
      "---------------------------------------------------------------------------------\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  col_name  feature_importance_vals\n",
      "2     lgbm                 9.598010\n",
      "1       rf                 4.530714\n",
      "0      dnn                 2.256583\n",
      "3      xgb                 1.921904\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 576x223.2 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print('---------------------------------------------------------------------------------')\n",
    "print('Generating SHAP explanation')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('')\n",
    "\n",
    "with open(output_file_name, \"a\") as f:print('LGBM FEATURE IMPORTANCE',file = f)\n",
    "\n",
    "    #START TIMER MODEL\n",
    "start = time.time()\n",
    "\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Generating explainer')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('')\n",
    "test = X_test_01\n",
    "train = X_train_01\n",
    "# ## Summary Bar Plot Global\n",
    "start_index = 0\n",
    "end_index = 250\n",
    "# test.pop('Label')\n",
    "# test.pop('is_train')\n",
    "# print(label2)\n",
    "\n",
    "\n",
    "explainer = shap.TreeExplainer(lgbm)\n",
    "\n",
    "shap_values = explainer.shap_values(test[start_index:end_index])\n",
    "shap_obj = explainer(test[start_index:end_index])\n",
    "shap.summary_plot(shap_values = shap_values,\n",
    "                  features = test[start_index:end_index],\n",
    "                show=False)\n",
    "# plt.clf()\n",
    "\n",
    "# if feature_selection_bit == 1 # On\n",
    "# pick_prob = 0 # set equal one to choose the dataset with probabilities, set to 0 to choose one with the classes.\n",
    "if pick_prob == 1:\n",
    "  plt.savefig('LGBM_SHAP_NSL_prob_01.png')\n",
    "elif pick_prob == 0:\n",
    "  plt.savefig('LGBM_SHAP_NSL_class_01.png')\n",
    "\n",
    "else: None\n",
    "plt.clf()\n",
    "\n",
    "\n",
    "vals= np.abs(shap_values).mean(1)\n",
    "feature_importance = pd.DataFrame(list(zip(train.columns, sum(vals))), columns=['col_name','feature_importance_vals'])\n",
    "feature_importance.sort_values(by=['feature_importance_vals'], ascending=False,inplace=True)\n",
    "feature_importance.head()\n",
    "print(feature_importance.to_string())\n",
    "with open(output_file_name, \"a\") as f:print('Feature Importance: ',feature_importance.to_string(),file = f)\n",
    "\n",
    "\n",
    "\n",
    "end = time.time()\n",
    "with open(output_file_name, \"a\") as f:print('ELAPSE TIME LIME GLOBAL: ',(end - start)/60, 'min',file = f)\n",
    "print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "\n",
    "print('---------------------------------------------------------------------------------')\n",
    "# feature_importance_vals = 'feature_importance_vals'  # Replace with the name of the column you want to extract\n",
    "feature_val = feature_importance['feature_importance_vals'].tolist()\n",
    "\n",
    "# col_name = 'col_name'  # Replace with the name of the column you want to extract\n",
    "feature_name = feature_importance['col_name'].tolist()\n",
    "\n",
    "\n",
    "# for item1, item2 in zip(feature_name, feature_val):\n",
    "#     print(item1, item2)\n",
    "\n",
    "\n",
    "# Use zip to combine the two lists, sort based on list1, and then unzip them\n",
    "zipped_lists = list(zip(feature_name, feature_val))\n",
    "zipped_lists.sort(key=lambda x: x[1],reverse=True)\n",
    "\n",
    "# Convert the sorted result back into separate lists\n",
    "sorted_list1, sorted_list2 = [list(x) for x in zip(*zipped_lists)]\n",
    "\n",
    "# for k in sorted_list1:\n",
    "#   with open(output_file_name, \"a\") as f: print(\"df.pop('\",k,\"')\", sep='', file = f)\n",
    "\n",
    "# with open(output_file_name, \"a\") as f:print(\"Trial_ =[\", file = f)\n",
    "# for k in sorted_list1:\n",
    "#   with open(output_file_name, \"a\") as f:print(\"'\",k,\"',\", sep='', file = f)\n",
    "# with open(output_file_name, \"a\") as f:print(\"]\", file = f)\n",
    "\n",
    "print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Generating SHAP explanation\n",
      "---------------------------------------------------------------------------------\n",
      "\n",
      "---------------------------------------------------------------------------------\n",
      "Generating explainer\n",
      "---------------------------------------------------------------------------------\n",
      "\n",
      "  col_name  feature_importance_vals\n",
      "2     lgbm                 0.670630\n",
      "3      xgb                 0.418141\n",
      "1       rf                 0.106782\n",
      "0      dnn                 0.024627\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 576x223.2 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print('---------------------------------------------------------------------------------')\n",
    "print('Generating SHAP explanation')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('')\n",
    "\n",
    "with open(output_file_name, \"a\") as f:print('RF FEATURE IMPORTANCE',file = f)\n",
    "\n",
    "    #START TIMER MODEL\n",
    "start = time.time()\n",
    "\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Generating explainer')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('')\n",
    "test = X_test_01\n",
    "train = X_train_01\n",
    "# ## Summary Bar Plot Global\n",
    "start_index = 0\n",
    "end_index = 250\n",
    "# test.pop('Label')\n",
    "# test.pop('is_train')\n",
    "# print(label2)\n",
    "\n",
    "\n",
    "explainer = shap.TreeExplainer(rf)\n",
    "\n",
    "shap_values = explainer.shap_values(test[start_index:end_index])\n",
    "shap_obj = explainer(test[start_index:end_index])\n",
    "shap.summary_plot(shap_values = shap_values,\n",
    "                  features = test[start_index:end_index],\n",
    "                show=False)\n",
    "# plt.clf()\n",
    "\n",
    "# if feature_selection_bit == 1 # On\n",
    "# pick_prob = 0 # set equal one to choose the dataset with probabilities, set to 0 to choose one with the classes.\n",
    "if pick_prob == 1:\n",
    "  plt.savefig('RF_SHAP_NSL_prob_01.png')\n",
    "elif pick_prob == 0:\n",
    "  plt.savefig('RF_SHAP_NSL_class_01.png')\n",
    "\n",
    "else: None\n",
    "plt.clf()\n",
    "\n",
    "\n",
    "vals= np.abs(shap_values).mean(1)\n",
    "feature_importance = pd.DataFrame(list(zip(train.columns, sum(vals))), columns=['col_name','feature_importance_vals'])\n",
    "feature_importance.sort_values(by=['feature_importance_vals'], ascending=False,inplace=True)\n",
    "feature_importance.head()\n",
    "print(feature_importance.to_string())\n",
    "with open(output_file_name, \"a\") as f:print('Feature Importance: ',feature_importance.to_string(),file = f)\n",
    "\n",
    "\n",
    "\n",
    "end = time.time()\n",
    "with open(output_file_name, \"a\") as f:print('ELAPSE TIME LIME GLOBAL: ',(end - start)/60, 'min',file = f)\n",
    "print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "\n",
    "print('---------------------------------------------------------------------------------')\n",
    "# feature_importance_vals = 'feature_importance_vals'  # Replace with the name of the column you want to extract\n",
    "feature_val = feature_importance['feature_importance_vals'].tolist()\n",
    "\n",
    "# col_name = 'col_name'  # Replace with the name of the column you want to extract\n",
    "feature_name = feature_importance['col_name'].tolist()\n",
    "\n",
    "\n",
    "# for item1, item2 in zip(feature_name, feature_val):\n",
    "#     print(item1, item2)\n",
    "\n",
    "\n",
    "# Use zip to combine the two lists, sort based on list1, and then unzip them\n",
    "zipped_lists = list(zip(feature_name, feature_val))\n",
    "zipped_lists.sort(key=lambda x: x[1],reverse=True)\n",
    "\n",
    "# Convert the sorted result back into separate lists\n",
    "sorted_list1, sorted_list2 = [list(x) for x in zip(*zipped_lists)]\n",
    "\n",
    "# for k in sorted_list1:\n",
    "#   with open(output_file_name, \"a\") as f: print(\"df.pop('\",k,\"')\", sep='', file = f)\n",
    "\n",
    "# with open(output_file_name, \"a\") as f:print(\"Trial_ =[\", file = f)\n",
    "# for k in sorted_list1:\n",
    "#   with open(output_file_name, \"a\") as f:print(\"'\",k,\"',\", sep='', file = f)\n",
    "# with open(output_file_name, \"a\") as f:print(\"]\", file = f)\n",
    "\n",
    "print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
